From 8be110ebf92277e4ff6fc94af7c854438ed1ce1c Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sat, 15 Nov 2014 16:53:51 +0800 Subject: [PATCH] logrotate --- examples/tunasync.conf | 6 ++-- tunasync/btrfs_snapshot.py | 6 ++-- tunasync/jobs.py | 7 ++-- tunasync/loglimit.py | 71 +++++++++++++++++++++++++++++++++++++ tunasync/mirror_config.py | 11 ++++-- tunasync/mirror_provider.py | 40 ++++++++++++--------- 6 files changed, 115 insertions(+), 26 deletions(-) create mode 100644 tunasync/loglimit.py diff --git a/examples/tunasync.conf b/examples/tunasync.conf index cf94012..a9edb43 100644 --- a/examples/tunasync.conf +++ b/examples/tunasync.conf @@ -30,20 +30,20 @@ name = "arch1" provider = "shell" command = "sleep 10" local_dir = "/mnt/sdb1/mirror/archlinux/current/" -log_file = "/dev/null" +# log_file = "/dev/null" [[mirrors]] name = "arch2" provider = "shell" command = "sleep 20" local_dir = "/mnt/sdb1/mirror/archlinux/current/" -log_file = "/dev/null" +# log_file = "/dev/null" [[mirrors]] name = "arch4" provider = "shell" command = "./shell_provider.sh" -log_file = "/tmp/arch4-{date}.log" +# log_file = "/tmp/arch4-{date}.log" use_btrfs = false # vim: ft=toml diff --git a/tunasync/btrfs_snapshot.py b/tunasync/btrfs_snapshot.py index 91b179b..163e0c7 100644 --- a/tunasync/btrfs_snapshot.py +++ b/tunasync/btrfs_snapshot.py @@ -17,12 +17,14 @@ class BtrfsHook(JobHook): self.working_dir = working_dir self.gc_dir = gc_dir - def before_job(self, *args, **kwargs): + def before_job(self, ctx={}, *args, **kwargs): self._create_working_snapshot() + ctx['current_dir'] = self.working_dir - def after_job(self, status=None, *args, **kwargs): + def after_job(self, status=None, ctx={}, *args, **kwargs): if status == "success": self._commit_changes() + ctx['current_dir'] = self.service_dir def _ensure_subvolume(self): # print(self.service_dir) diff --git a/tunasync/jobs.py b/tunasync/jobs.py index 88b51ae..da10b3c 100644 --- a/tunasync/jobs.py +++ b/tunasync/jobs.py @@ -30,9 +30,10 @@ def run_job(sema, child_q, manager_q, provider, **settings): status = "syncing" manager_q.put(("UPDATE", (provider.name, status))) + ctx = {} # put context info in it try: for hook in provider.hooks: - hook.before_job(name=provider.name) + hook.before_job(provider=provider, ctx=ctx) except Exception: import traceback traceback.print_exc() @@ -40,7 +41,7 @@ def run_job(sema, child_q, manager_q, provider, **settings): else: for retry in range(max_retry): print("start syncing {}, retry: {}".format(provider.name, retry)) - provider.run() + provider.run(ctx=ctx) status = "success" try: @@ -53,7 +54,7 @@ def run_job(sema, child_q, manager_q, provider, **settings): try: for hook in provider.hooks[::-1]: - hook.after_job(name=provider.name, status=status) + hook.after_job(provider=provider, status=status, ctx=ctx) except Exception: import traceback traceback.print_exc() diff --git a/tunasync/loglimit.py b/tunasync/loglimit.py new file mode 100644 index 0000000..86a72a8 --- /dev/null +++ b/tunasync/loglimit.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python2 +# -*- coding:utf-8 -*- +import sh +import os +from .hook import JobHook +from datetime import datetime + + +class LogLimitHook(JobHook): + + def __init__(self, limit=10): + self.limit = limit + + def before_job(self, provider, ctx={}, *args, **kwargs): + log_dir = provider.log_dir + self.ensure_log_dir(log_dir) + log_file = provider.log_file.format( + date=datetime.now().strftime("%Y-%m-%d_%H-%M")) + ctx['log_file'] = log_file + if log_file == "/dev/null": + return + + log_link = os.path.join(log_dir, "latest") + + lfiles = [os.path.join(log_dir, lfile) + for lfile in os.listdir(log_dir) + if lfile.startswith(provider.name)] + + lfiles_set = set(lfiles) + # sort to get the newest 10 files + lfiles_ts = sorted( + [(os.path.getmtime(lfile), lfile) for lfile in lfiles], + key=lambda x: x[0], + reverse=True) + lfiles_keep = set([x[1] for x in lfiles_ts[:self.limit]]) + lfiles_rm = lfiles_set - lfiles_keep + # remove old files + for lfile in lfiles_rm: + try: + sh.rm(lfile) + except: + pass + + # create a soft link + if log_link != log_file: + if os.path.exists(log_link): + try: + sh.rm(log_link) + except: + return + try: + sh.ln('-s', log_file, log_link) + except: + return + + def after_job(self, status=None, ctx={}, *args, **kwargs): + log_file = ctx.get('log_file', None) + if log_file == "/dev/null": + return + if status == "fail": + log_file_save = log_file + ".fail" + try: + sh.mv(log_file, log_file_save) + except: + pass + + def ensure_log_dir(self, log_dir): + if not os.path.exists(log_dir): + sh.mkdir("-p", log_dir) + +# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/mirror_config.py b/tunasync/mirror_config.py index 42e5cf5..ebc2078 100644 --- a/tunasync/mirror_config.py +++ b/tunasync/mirror_config.py @@ -3,6 +3,7 @@ import os from .mirror_provider import RsyncProvider, ShellProvider from .btrfs_snapshot import BtrfsHook +from .loglimit import LogLimitHook class MirrorConfig(object): @@ -38,10 +39,13 @@ class MirrorConfig(object): assert isinstance(self.options["interval"], int) - log_dir = self._popt["global"]["log_dir"] + log_dir = self.options.get( + "log_dir", self._popt["global"]["log_dir"]) if "log_file" not in self.options: self.options["log_file"] = os.path.join( - log_dir, self.name, "{date}.log") + log_dir, self.name, self.name + "_{date}.log") + + self.log_dir = os.path.dirname(self.log_file) if "use_btrfs" not in self.options: self.options["use_btrfs"] = self._parent.use_btrfs @@ -59,6 +63,7 @@ class MirrorConfig(object): self.name, self.upstream, self.local_dir, + self.log_dir, self.use_ipv6, self.password, self.exclude_file, @@ -71,6 +76,7 @@ class MirrorConfig(object): self.name, self.command, self.local_dir, + self.log_dir, self.log_file, self.interval, hooks @@ -105,6 +111,7 @@ class MirrorConfig(object): ) hooks.append(BtrfsHook(service_dir, working_dir, gc_dir)) + hooks.append(LogLimitHook()) return hooks # vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/mirror_provider.py b/tunasync/mirror_provider.py index 6f43cc3..6c889f1 100644 --- a/tunasync/mirror_provider.py +++ b/tunasync/mirror_provider.py @@ -10,21 +10,32 @@ class MirrorProvider(object): Mirror method class, can be `rsync', `debmirror', etc. ''' - def __init__(self, name, local_dir, log_file="/dev/null", + def __init__(self, name, local_dir, log_dir, log_file="/dev/null", interval=120, hooks=[]): self.name = name self.local_dir = local_dir self.log_file = log_file + self.log_dir = log_dir self.interval = interval self.hooks = hooks self.p = None + # deprecated def ensure_log_dir(self): log_dir = os.path.dirname(self.log_file) if not os.path.exists(log_dir): sh.mkdir("-p", log_dir) - def run(self): + def get_log_file(self, ctx={}): + if 'log_file' in ctx: + log_file = ctx['log_file'] + else: + now = datetime.now().strftime("%Y-%m-%d_%H") + log_file = self.log_file.format(date=now) + ctx['log_file'] = log_file + return log_file + + def run(self, ctx={}): raise NotImplementedError("run method should be implemented") def terminate(self): @@ -44,10 +55,10 @@ class RsyncProvider(MirrorProvider): _default_options = \ "-aHvh --stats --delete-after --timeout=120 --contimeout=120" - def __init__(self, name, upstream_url, local_dir, useIPv6=True, - password=None, exclude_file=None, log_file="/dev/null", - interval=120, hooks=[]): - super(RsyncProvider, self).__init__(name, local_dir, log_file, + def __init__(self, name, upstream_url, local_dir, log_dir, + useIPv6=True, password=None, exclude_file=None, + log_file="/dev/null", interval=120, hooks=[]): + super(RsyncProvider, self).__init__(name, local_dir, log_dir, log_file, interval, hooks) self.upstream_url = upstream_url @@ -69,14 +80,12 @@ class RsyncProvider(MirrorProvider): return _options - def run(self): - self.ensure_log_dir() + def run(self, ctx={}): _args = self.options _args.append(self.upstream_url) _args.append(self.local_dir) - now = datetime.now().strftime("%Y-%m-%d_%H") - log_file = self.log_file.format(date=now) + log_file = self.get_log_file(ctx) new_env = os.environ.copy() if self.password is not None: new_env["RSYNC_PASSWORD"] = self.password @@ -87,17 +96,16 @@ class RsyncProvider(MirrorProvider): class ShellProvider(MirrorProvider): - def __init__(self, name, command, local_dir, + def __init__(self, name, command, local_dir, log_dir, log_file="/dev/null", interval=120, hooks=[]): - super(ShellProvider, self).__init__(name, local_dir, log_file, + super(ShellProvider, self).__init__(name, local_dir, log_dir, log_file, interval, hooks) self.command = command.split() - def run(self): - self.ensure_log_dir() - now = datetime.now().strftime("%Y-%m-%d_%H") - log_file = self.log_file.format(date=now) + def run(self, ctx={}): + + log_file = self.get_log_file(ctx) new_env = os.environ.copy() new_env["TUNASYNC_MIRROR_NAME"] = self.name