switch to toml

This commit is contained in:
bigeagle 2014-10-24 00:21:38 +08:00
parent 091f151956
commit d595af6755
3 changed files with 68 additions and 90 deletions

39
examples/tunasync.conf Normal file
View File

@ -0,0 +1,39 @@
[global]
log_dir = "/var/log/tunasync"
# mirror_root = /srv/mirror_disk
mirror_root = "/mnt/sdb1/mirror"
use_btrfs = false
local_dir = "{mirror_root}/_working/{mirror_name}/"
# maximum numbers of running jobs
concurrent = 2
# interval in minutes
interval = 1
max_retry = 2
[btrfs]
service_dir = "{mirror_root}/_current/{mirror_name}"
working_dir = "{mirror_root}/_working/{mirror_name}"
gc_dir = "{mirror_root}/_gc/{mirror_name}_{{timestamp}}"
[[mirrors]]
name = "archlinux"
provider = "rsync"
upstream = "rsync://mirror.us.leaseweb.net/archlinux/"
log_file = "/tmp/archlinux-{date}.log"
use_ipv6 = true
[[mirrors]]
name = "arch2"
provider = "shell"
command = "sleep 20"
local_dir = "/mnt/sdb1/mirror/archlinux/current/"
log_file = "/tmp/arch2-{date}.log"
[[mirrors]]
name = "arch4"
provider = "shell"
command = "./shell_provider.sh"
log_file = "/tmp/arch4-{date}.log"
use_btrfs = false
# vim: ft=toml

View File

@ -1,49 +0,0 @@
[global]
log_dir = /var/log/tunasync
; mirror_root = /srv/mirror_disk
mirror_root = /mnt/sdb1/mirror
use_btrfs = no
local_dir = {mirror_root}/{mirror_name}/_working
; maximum numbers of running jobs
concurrent = 2
; interval in minutes
interval = 1
max_retry = 2
[btrfs]
service_dir = {mirror_root}/{mirror_name}/_current
working_dir = {mirror_root}/{mirror_name}/_working
gc_dir = {mirror_root}/{mirror_name}/_gc_{{timestamp}}
# rmirror:archlinux]
# provider = rsync
# upstream = rsync://mirror.us.leaseweb.net/archlinux/
# log_file = /tmp/archlinux-{date}.log
# use_ipv6 = yes
[mirror:archlinux]
provider = shell
command = sleep 20
local_dir = /mnt/sdb1/mirror/archlinux/current/
log_file = /tmp/archlinux-{date}.log
[mirror:arch2]
provider = shell
command = sleep 5
log_file = /tmp/arch2-{date}.log
use_btrfs = no
[mirror:arch3]
provider = shell
command = ./shell_provider.sh
log_file = /tmp/arch3-{date}.log
use_btrfs = no
[mirror:arch4]
provider = shell
command = ./shell_provider.sh
log_file = /tmp/arch4-{date}.log
use_btrfs = no

View File

@ -1,9 +1,9 @@
#!/usr/bin/env python2 #!/usr/bin/env python2
# -*- coding:utf-8 -*- # -*- coding:utf-8 -*-
import ConfigParser
import os.path import os.path
import signal import signal
import sys import sys
import toml
from multiprocessing import Process, Semaphore, Queue from multiprocessing import Process, Semaphore, Queue
from . import jobs from . import jobs
@ -16,13 +16,10 @@ class MirrorConfig(object):
_valid_providers = set(("rsync", "debmirror", "shell", )) _valid_providers = set(("rsync", "debmirror", "shell", ))
def __init__(self, parent, name, cfgParser, section): def __init__(self, parent, options):
self._parent = parent self._parent = parent
self._cp = cfgParser self._popt = self._parent._settings
self._sec = section self.options = dict(options.items()) # copy
self.name = name
self.options = dict(self._cp.items(self._sec))
self._validate() self._validate()
def _validate(self): def _validate(self):
@ -31,37 +28,31 @@ class MirrorConfig(object):
if provider == "rsync": if provider == "rsync":
assert "upstream" in self.options assert "upstream" in self.options
if "use_ipv6" in self.options:
self.options["use_ipv6"] = self._cp.getboolean(self._sec,
"use_ipv6")
elif provider == "shell": elif provider == "shell":
assert "command" in self.options assert "command" in self.options
local_dir_tmpl = self.options.get( local_dir_tmpl = self.options.get(
"local_dir", self._cp.get("global", "local_dir")) "local_dir", self._popt["global"]["local_dir"])
self.options["local_dir"] = local_dir_tmpl.format( self.options["local_dir"] = local_dir_tmpl.format(
mirror_root=self._cp.get("global", "mirror_root"), mirror_root=self._popt["global"]["mirror_root"],
mirror_name=self.name, mirror_name=self.name,
) )
self.options["interval"] = int( if "interval" not in self.options:
self.options.get("interval", self.options["interval"] = self._popt["global"]["interval"]
self._cp.getint("global", "interval"))
)
log_dir = self._cp.get("global", "log_dir") assert isinstance(self.options["interval"], int)
self.options["log_file"] = self.options.get(
"log_file",
os.path.join(log_dir, self.name, "{date}.log")
)
try: log_dir = self._popt["global"]["log_dir"]
self.options["use_btrfs"] = self._cp.getboolean( if "log_file" not in self.options:
self._sec, "use_btrfs") self.options["log_file"] = os.path.join(
except ConfigParser.NoOptionError: log_dir, self.name, "{date}.log")
if "use_btrfs" not in self.options:
self.options["use_btrfs"] = self._parent.use_btrfs self.options["use_btrfs"] = self._parent.use_btrfs
assert self.options["use_btrfs"] in (True, False)
def __getattr__(self, key): def __getattr__(self, key):
if key in self.__dict__: if key in self.__dict__:
@ -137,25 +128,23 @@ class TUNASync(object):
def read_config(self, config_file): def read_config(self, config_file):
self._config_file = config_file self._config_file = config_file
self._settings = ConfigParser.ConfigParser() with open(self._config_file) as f:
self._settings.read(config_file) self._settings = toml.loads(f.read())
self._inited = True self._inited = True
self._mirrors = {} self._mirrors = {}
self._providers = {} self._providers = {}
self.processes = {} self.processes = {}
self.semaphore = Semaphore(self._settings.getint("global", "concurrent")) self.semaphore = Semaphore(self._settings["global"]["concurrent"])
self.channel = Queue() self.channel = Queue()
self._hooks = [] self._hooks = []
self.mirror_root = self._settings.get("global", "mirror_root") self.mirror_root = self._settings["global"]["mirror_root"]
self.use_btrfs = self._settings.getboolean("global", "use_btrfs")
self.btrfs_service_dir_tmpl = self._settings.get( self.use_btrfs = self._settings["global"]["use_btrfs"]
"btrfs", "service_dir") self.btrfs_service_dir_tmpl = self._settings["btrfs"]["service_dir"]
self.btrfs_working_dir_tmpl = self._settings.get( self.btrfs_working_dir_tmpl = self._settings["btrfs"]["working_dir"]
"btrfs", "working_dir") self.btrfs_gc_dir_tmpl = self._settings["btrfs"]["gc_dir"]
self.btrfs_gc_dir_tmpl = self._settings.get(
"btrfs", "gc_dir")
def add_hook(self, h): def add_hook(self, h):
assert isinstance(h, JobHook) assert isinstance(h, JobHook)
@ -169,12 +158,11 @@ class TUNASync(object):
if self._mirrors: if self._mirrors:
return self._mirrors return self._mirrors
for section in filter(lambda s: s.startswith("mirror:"), for mirror_opt in self._settings["mirrors"]:
self._settings.sections()): name = mirror_opt["name"]
_, name = section.split(":")
self._mirrors[name] = \ self._mirrors[name] = \
MirrorConfig(self, name, self._settings, section) MirrorConfig(self, mirror_opt)
return self._mirrors return self._mirrors
@property @property