switch to toml

This commit is contained in:
bigeagle 2014-10-24 00:21:38 +08:00
parent 091f151956
commit d595af6755
3 changed files with 68 additions and 90 deletions

39
examples/tunasync.conf Normal file
View File

@ -0,0 +1,39 @@
[global]
log_dir = "/var/log/tunasync"
# mirror_root = /srv/mirror_disk
mirror_root = "/mnt/sdb1/mirror"
use_btrfs = false
local_dir = "{mirror_root}/_working/{mirror_name}/"
# maximum numbers of running jobs
concurrent = 2
# interval in minutes
interval = 1
max_retry = 2
[btrfs]
service_dir = "{mirror_root}/_current/{mirror_name}"
working_dir = "{mirror_root}/_working/{mirror_name}"
gc_dir = "{mirror_root}/_gc/{mirror_name}_{{timestamp}}"
[[mirrors]]
name = "archlinux"
provider = "rsync"
upstream = "rsync://mirror.us.leaseweb.net/archlinux/"
log_file = "/tmp/archlinux-{date}.log"
use_ipv6 = true
[[mirrors]]
name = "arch2"
provider = "shell"
command = "sleep 20"
local_dir = "/mnt/sdb1/mirror/archlinux/current/"
log_file = "/tmp/arch2-{date}.log"
[[mirrors]]
name = "arch4"
provider = "shell"
command = "./shell_provider.sh"
log_file = "/tmp/arch4-{date}.log"
use_btrfs = false
# vim: ft=toml

View File

@ -1,49 +0,0 @@
[global]
log_dir = /var/log/tunasync
; mirror_root = /srv/mirror_disk
mirror_root = /mnt/sdb1/mirror
use_btrfs = no
local_dir = {mirror_root}/{mirror_name}/_working
; maximum numbers of running jobs
concurrent = 2
; interval in minutes
interval = 1
max_retry = 2
[btrfs]
service_dir = {mirror_root}/{mirror_name}/_current
working_dir = {mirror_root}/{mirror_name}/_working
gc_dir = {mirror_root}/{mirror_name}/_gc_{{timestamp}}
# rmirror:archlinux]
# provider = rsync
# upstream = rsync://mirror.us.leaseweb.net/archlinux/
# log_file = /tmp/archlinux-{date}.log
# use_ipv6 = yes
[mirror:archlinux]
provider = shell
command = sleep 20
local_dir = /mnt/sdb1/mirror/archlinux/current/
log_file = /tmp/archlinux-{date}.log
[mirror:arch2]
provider = shell
command = sleep 5
log_file = /tmp/arch2-{date}.log
use_btrfs = no
[mirror:arch3]
provider = shell
command = ./shell_provider.sh
log_file = /tmp/arch3-{date}.log
use_btrfs = no
[mirror:arch4]
provider = shell
command = ./shell_provider.sh
log_file = /tmp/arch4-{date}.log
use_btrfs = no

View File

@ -1,9 +1,9 @@
#!/usr/bin/env python2
# -*- coding:utf-8 -*-
import ConfigParser
import os.path
import signal
import sys
import toml
from multiprocessing import Process, Semaphore, Queue
from . import jobs
@ -16,13 +16,10 @@ class MirrorConfig(object):
_valid_providers = set(("rsync", "debmirror", "shell", ))
def __init__(self, parent, name, cfgParser, section):
def __init__(self, parent, options):
self._parent = parent
self._cp = cfgParser
self._sec = section
self.name = name
self.options = dict(self._cp.items(self._sec))
self._popt = self._parent._settings
self.options = dict(options.items()) # copy
self._validate()
def _validate(self):
@ -31,37 +28,31 @@ class MirrorConfig(object):
if provider == "rsync":
assert "upstream" in self.options
if "use_ipv6" in self.options:
self.options["use_ipv6"] = self._cp.getboolean(self._sec,
"use_ipv6")
elif provider == "shell":
assert "command" in self.options
local_dir_tmpl = self.options.get(
"local_dir", self._cp.get("global", "local_dir"))
"local_dir", self._popt["global"]["local_dir"])
self.options["local_dir"] = local_dir_tmpl.format(
mirror_root=self._cp.get("global", "mirror_root"),
mirror_root=self._popt["global"]["mirror_root"],
mirror_name=self.name,
)
self.options["interval"] = int(
self.options.get("interval",
self._cp.getint("global", "interval"))
)
if "interval" not in self.options:
self.options["interval"] = self._popt["global"]["interval"]
log_dir = self._cp.get("global", "log_dir")
self.options["log_file"] = self.options.get(
"log_file",
os.path.join(log_dir, self.name, "{date}.log")
)
assert isinstance(self.options["interval"], int)
try:
self.options["use_btrfs"] = self._cp.getboolean(
self._sec, "use_btrfs")
except ConfigParser.NoOptionError:
log_dir = self._popt["global"]["log_dir"]
if "log_file" not in self.options:
self.options["log_file"] = os.path.join(
log_dir, self.name, "{date}.log")
if "use_btrfs" not in self.options:
self.options["use_btrfs"] = self._parent.use_btrfs
assert self.options["use_btrfs"] in (True, False)
def __getattr__(self, key):
if key in self.__dict__:
@ -137,25 +128,23 @@ class TUNASync(object):
def read_config(self, config_file):
self._config_file = config_file
self._settings = ConfigParser.ConfigParser()
self._settings.read(config_file)
with open(self._config_file) as f:
self._settings = toml.loads(f.read())
self._inited = True
self._mirrors = {}
self._providers = {}
self.processes = {}
self.semaphore = Semaphore(self._settings.getint("global", "concurrent"))
self.semaphore = Semaphore(self._settings["global"]["concurrent"])
self.channel = Queue()
self._hooks = []
self.mirror_root = self._settings.get("global", "mirror_root")
self.use_btrfs = self._settings.getboolean("global", "use_btrfs")
self.btrfs_service_dir_tmpl = self._settings.get(
"btrfs", "service_dir")
self.btrfs_working_dir_tmpl = self._settings.get(
"btrfs", "working_dir")
self.btrfs_gc_dir_tmpl = self._settings.get(
"btrfs", "gc_dir")
self.mirror_root = self._settings["global"]["mirror_root"]
self.use_btrfs = self._settings["global"]["use_btrfs"]
self.btrfs_service_dir_tmpl = self._settings["btrfs"]["service_dir"]
self.btrfs_working_dir_tmpl = self._settings["btrfs"]["working_dir"]
self.btrfs_gc_dir_tmpl = self._settings["btrfs"]["gc_dir"]
def add_hook(self, h):
assert isinstance(h, JobHook)
@ -169,12 +158,11 @@ class TUNASync(object):
if self._mirrors:
return self._mirrors
for section in filter(lambda s: s.startswith("mirror:"),
self._settings.sections()):
_, name = section.split(":")
for mirror_opt in self._settings["mirrors"]:
name = mirror_opt["name"]
self._mirrors[name] = \
MirrorConfig(self, name, self._settings, section)
MirrorConfig(self, mirror_opt)
return self._mirrors
@property