diff --git a/tests/mirrors/aosp.conf b/tests/mirrors/aosp.conf new file mode 100644 index 0000000..b4ce206 --- /dev/null +++ b/tests/mirrors/aosp.conf @@ -0,0 +1,12 @@ +[[mirrors]] +name = "AOSP" +provider = "command" +command = "/tmp/tunasync/bin/myrsync2.sh" +upstream = "https://aosp.google.com/" +interval = 2 +mirror_dir = "/tmp/tunasync/git/AOSP" +role = "slave" + [mirrors.env] + REPO = "/usr/local/bin/aosp-repo" + +# vim: ft=toml diff --git a/tests/mirrors/deb_fedora.conf b/tests/mirrors/deb_fedora.conf new file mode 100644 index 0000000..f4b1f3e --- /dev/null +++ b/tests/mirrors/deb_fedora.conf @@ -0,0 +1,18 @@ +[[mirrors]] +name = "debian" +command = "/tmp/tunasync/bin/myrsync.sh" +provider = "two-stage-rsync" +stage1_profile = "debian" +upstream = "rsync://ftp.debian.org/debian/" +use_ipv6 = true + + +[[mirrors]] +name = "fedora" +command = "/tmp/tunasync/bin/myrsync.sh" +provider = "rsync" +upstream = "rsync://ftp.fedoraproject.org/fedora/" +use_ipv6 = true +exclude_file = "/etc/tunasync.d/fedora-exclude.txt" + +# vim: ft=toml diff --git a/tests/worker.conf b/tests/worker.conf index 42550a0..3eba460 100644 --- a/tests/worker.conf +++ b/tests/worker.conf @@ -22,33 +22,7 @@ listen_port = 6000 ssl_cert = "worker.crt" ssl_key = "worker.key" -[[mirrors]] -name = "AOSP" -provider = "command" -command = "/tmp/tunasync/bin/myrsync2.sh" -upstream = "https://aosp.google.com/" -interval = 2 -mirror_dir = "/tmp/tunasync/git/AOSP" -role = "slave" - [mirrors.env] - REPO = "/usr/local/bin/aosp-repo" - -[[mirrors]] -name = "debian" -command = "/tmp/tunasync/bin/myrsync.sh" -provider = "two-stage-rsync" -stage1_profile = "debian" -upstream = "rsync://ftp.debian.org/debian/" -use_ipv6 = true - - -[[mirrors]] -name = "fedora" -command = "/tmp/tunasync/bin/myrsync.sh" -provider = "rsync" -upstream = "rsync://ftp.fedoraproject.org/fedora/" -use_ipv6 = true -exclude_file = "/etc/tunasync.d/fedora-exclude.txt" - +[include] +include_mirrors = "mirrors/*.conf" # vim: ft=toml diff --git a/worker/config.go b/worker/config.go index 8515a83..0d8a20b 100644 --- a/worker/config.go +++ b/worker/config.go @@ -3,6 +3,7 @@ package worker import ( "errors" "os" + "path/filepath" "github.com/BurntSushi/toml" ) @@ -30,12 +31,13 @@ func (p *providerEnum) UnmarshalText(text []byte) error { return nil } -// Worker config options +// Config represents worker config options type Config struct { Global globalConfig `toml:"global"` Manager managerConfig `toml:"manager"` Server serverConfig `toml:"server"` Cgroup cgroupConfig `toml:"cgroup"` + Include includeConfig `toml:"include"` Mirrors []mirrorConfig `toml:"mirrors"` } @@ -67,6 +69,14 @@ type cgroupConfig struct { Group string `toml:"group"` } +type includeConfig struct { + IncludeMirrors string `toml:"include_mirrors"` +} + +type includedMirrorConfig struct { + Mirrors []mirrorConfig `toml:"mirrors"` +} + type mirrorConfig struct { Name string `toml:"name"` Provider providerEnum `toml:"provider"` @@ -98,5 +108,22 @@ func LoadConfig(cfgFile string) (*Config, error) { logger.Errorf(err.Error()) return nil, err } + + if cfg.Include.IncludeMirrors != "" { + includedFiles, err := filepath.Glob(cfg.Include.IncludeMirrors) + if err != nil { + logger.Errorf(err.Error()) + return nil, err + } + for _, f := range includedFiles { + var incMirCfg includedMirrorConfig + if _, err := toml.DecodeFile(f, &incMirCfg); err != nil { + logger.Errorf(err.Error()) + return nil, err + } + cfg.Mirrors = append(cfg.Mirrors, incMirCfg.Mirrors...) + } + } + return cfg, nil } diff --git a/worker/config_test.go b/worker/config_test.go index 45762d9..1fb4488 100644 --- a/worker/config_test.go +++ b/worker/config_test.go @@ -1,8 +1,10 @@ package worker import ( + "fmt" "io/ioutil" "os" + "path/filepath" "testing" . "github.com/smartystreets/goconvey/convey" @@ -45,7 +47,6 @@ stage1_profile = "debian" upstream = "rsync://ftp.debian.org/debian/" use_ipv6 = true - [[mirrors]] name = "fedora" provider = "rsync" @@ -66,10 +67,47 @@ exec_on_failure = "bash -c 'echo ${TUNASYNC_JOB_EXIT_STATUS} > ${TUNASYNC_WORKIN So(err, ShouldEqual, nil) defer os.Remove(tmpfile.Name()) + tmpDir, err := ioutil.TempDir("", "tunasync") + So(err, ShouldBeNil) + defer os.RemoveAll(tmpDir) + + incSection := fmt.Sprintf( + "\n[include]\n"+ + "include_mirrors = \"%s/*.conf\"", + tmpDir, + ) + + cfgBlob = cfgBlob + incSection + err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644) So(err, ShouldEqual, nil) defer tmpfile.Close() + incBlob1 := ` +[[mirrors]] +name = "debian-cd" +provider = "two-stage-rsync" +stage1_profile = "debian" +use_ipv6 = true + +[[mirrors]] +name = "debian-security" +provider = "two-stage-rsync" +stage1_profile = "debian" +use_ipv6 = true + ` + incBlob2 := ` +[[mirrors]] +name = "ubuntu" +provider = "two-stage-rsync" +stage1_profile = "debian" +use_ipv6 = true + ` + err = ioutil.WriteFile(filepath.Join(tmpDir, "debian.conf"), []byte(incBlob1), 0644) + So(err, ShouldEqual, nil) + err = ioutil.WriteFile(filepath.Join(tmpDir, "ubuntu.conf"), []byte(incBlob2), 0644) + So(err, ShouldEqual, nil) + cfg, err := LoadConfig(tmpfile.Name()) So(err, ShouldBeNil) So(cfg.Global.Name, ShouldEqual, "test_worker") @@ -97,7 +135,18 @@ exec_on_failure = "bash -c 'echo ${TUNASYNC_JOB_EXIT_STATUS} > ${TUNASYNC_WORKIN So(m.Provider, ShouldEqual, provRsync) So(m.ExcludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt") - So(len(cfg.Mirrors), ShouldEqual, 3) + m = cfg.Mirrors[3] + So(m.Name, ShouldEqual, "debian-cd") + So(m.MirrorDir, ShouldEqual, "") + So(m.Provider, ShouldEqual, provTwoStageRsync) + + m = cfg.Mirrors[4] + So(m.Name, ShouldEqual, "debian-security") + + m = cfg.Mirrors[5] + So(m.Name, ShouldEqual, "ubuntu") + + So(len(cfg.Mirrors), ShouldEqual, 6) }) Convey("Providers can be inited from a valid config file", t, func() {