mirror of
https://github.com/tuna/tunasync.git
synced 2025-04-20 20:22:46 +00:00
Merge pull request #114 from tuna/nest_mirror
Support nested mirror config
This commit is contained in:
Yuxiang Zhang
GitHub
commit
3fed3f1cf3
1
go.mod
1
go.mod
@ -9,6 +9,7 @@ require (
|
||||
github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27
|
||||
github.com/dennwc/btrfs v0.0.0-20190517175702-d917b30ff035
|
||||
github.com/gin-gonic/gin v1.5.0
|
||||
github.com/imdario/mergo v0.3.9
|
||||
github.com/mattn/goveralls v0.0.5 // indirect
|
||||
github.com/pkg/profile v1.4.0
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46
|
||||
|
||||
5
go.sum
5
go.sum
@ -29,6 +29,9 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
|
||||
github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
|
||||
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
|
||||
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
@ -38,7 +41,9 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/goveralls v0.0.5 h1:spfq8AyZ0cCk57Za6/juJ5btQxeE1FaEGMdfcI+XO48=
|
||||
github.com/mattn/goveralls v0.0.5/go.mod h1:Xg2LHi51faXLyKXwsndxiW6uxEEQT9+3sjGzzwU4xy0=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/pkg/profile v1.4.0 h1:uCmaf4vVbWAOZz36k1hrQD7ijGRzLwaME8Am/7a4jZI=
|
||||
github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE=
|
||||
|
||||
@ -6,6 +6,7 @@ import (
|
||||
"path/filepath"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/imdario/mergo"
|
||||
)
|
||||
|
||||
type providerEnum uint8
|
||||
@ -41,7 +42,8 @@ type Config struct {
|
||||
BtrfsSnapshot btrfsSnapshotConfig `toml:"btrfs_snapshot"`
|
||||
Docker dockerConfig `toml:"docker"`
|
||||
Include includeConfig `toml:"include"`
|
||||
Mirrors []mirrorConfig `toml:"mirrors"`
|
||||
MirrorsConf []mirrorConfig `toml:"mirrors"`
|
||||
Mirrors []mirrorConfig
|
||||
}
|
||||
|
||||
type globalConfig struct {
|
||||
@ -117,6 +119,7 @@ type mirrorConfig struct {
|
||||
Interval int `toml:"interval"`
|
||||
Retry int `toml:"retry"`
|
||||
MirrorDir string `toml:"mirror_dir"`
|
||||
MirrorSubDir string `toml:"mirror_subdir"`
|
||||
LogDir string `toml:"log_dir"`
|
||||
Env map[string]string `toml:"env"`
|
||||
Role string `toml:"role"`
|
||||
@ -148,6 +151,8 @@ type mirrorConfig struct {
|
||||
DockerOptions []string `toml:"docker_options"`
|
||||
|
||||
SnapshotPath string `toml:"snapshot_path"`
|
||||
|
||||
ChildMirrors []mirrorConfig `toml:"mirrors"`
|
||||
}
|
||||
|
||||
// LoadConfig loads configuration
|
||||
@ -174,9 +179,36 @@ func LoadConfig(cfgFile string) (*Config, error) {
|
||||
logger.Errorf(err.Error())
|
||||
return nil, err
|
||||
}
|
||||
cfg.Mirrors = append(cfg.Mirrors, incMirCfg.Mirrors...)
|
||||
cfg.MirrorsConf = append(cfg.MirrorsConf, incMirCfg.Mirrors...)
|
||||
}
|
||||
}
|
||||
|
||||
for _, m := range cfg.MirrorsConf {
|
||||
if err := recursiveMirrors(cfg, nil, m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func recursiveMirrors(cfg *Config, parent *mirrorConfig, mirror mirrorConfig) error {
|
||||
var curMir mirrorConfig
|
||||
if parent != nil {
|
||||
curMir = *parent
|
||||
}
|
||||
curMir.ChildMirrors = nil
|
||||
if err := mergo.Merge(&curMir, mirror, mergo.WithOverride); err != nil {
|
||||
return err
|
||||
}
|
||||
if mirror.ChildMirrors == nil {
|
||||
cfg.Mirrors = append(cfg.Mirrors, curMir)
|
||||
} else {
|
||||
for _, m := range mirror.ChildMirrors {
|
||||
if err := recursiveMirrors(cfg, &curMir, m); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -83,9 +83,9 @@ exec_on_failure = [
|
||||
tmpDir,
|
||||
)
|
||||
|
||||
cfgBlob = cfgBlob + incSection
|
||||
curCfgBlob := cfgBlob + incSection
|
||||
|
||||
err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644)
|
||||
err = ioutil.WriteFile(tmpfile.Name(), []byte(curCfgBlob), 0644)
|
||||
So(err, ShouldEqual, nil)
|
||||
defer tmpfile.Close()
|
||||
|
||||
@ -157,6 +157,102 @@ use_ipv6 = true
|
||||
So(len(cfg.Mirrors), ShouldEqual, 6)
|
||||
})
|
||||
|
||||
Convey("Everything should work on nested config file", t, func() {
|
||||
tmpfile, err := ioutil.TempFile("", "tunasync")
|
||||
So(err, ShouldEqual, nil)
|
||||
defer os.Remove(tmpfile.Name())
|
||||
|
||||
tmpDir, err := ioutil.TempDir("", "tunasync")
|
||||
So(err, ShouldBeNil)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
incSection := fmt.Sprintf(
|
||||
"\n[include]\n"+
|
||||
"include_mirrors = \"%s/*.conf\"",
|
||||
tmpDir,
|
||||
)
|
||||
|
||||
curCfgBlob := cfgBlob + incSection
|
||||
|
||||
err = ioutil.WriteFile(tmpfile.Name(), []byte(curCfgBlob), 0644)
|
||||
So(err, ShouldEqual, nil)
|
||||
defer tmpfile.Close()
|
||||
|
||||
incBlob1 := `
|
||||
[[mirrors]]
|
||||
name = "ipv6s"
|
||||
use_ipv6 = true
|
||||
[[mirrors.mirrors]]
|
||||
name = "debians"
|
||||
mirror_subdir = "debian"
|
||||
provider = "two-stage-rsync"
|
||||
stage1_profile = "debian"
|
||||
|
||||
[[mirrors.mirrors.mirrors]]
|
||||
name = "debian-security"
|
||||
upstream = "rsync://test.host/debian-security/"
|
||||
[[mirrors.mirrors.mirrors]]
|
||||
name = "ubuntu"
|
||||
stage1_profile = "ubuntu"
|
||||
upstream = "rsync://test.host2/ubuntu/"
|
||||
[[mirrors.mirrors]]
|
||||
name = "debian-cd"
|
||||
provider = "rsync"
|
||||
upstream = "rsync://test.host3/debian-cd/"
|
||||
`
|
||||
err = ioutil.WriteFile(filepath.Join(tmpDir, "nest.conf"), []byte(incBlob1), 0644)
|
||||
So(err, ShouldEqual, nil)
|
||||
|
||||
cfg, err := LoadConfig(tmpfile.Name())
|
||||
So(err, ShouldBeNil)
|
||||
So(cfg.Global.Name, ShouldEqual, "test_worker")
|
||||
So(cfg.Global.Interval, ShouldEqual, 240)
|
||||
So(cfg.Global.Retry, ShouldEqual, 3)
|
||||
So(cfg.Global.MirrorDir, ShouldEqual, "/data/mirrors")
|
||||
|
||||
So(cfg.Manager.APIBase, ShouldEqual, "https://127.0.0.1:5000")
|
||||
So(cfg.Server.Hostname, ShouldEqual, "worker1.example.com")
|
||||
|
||||
m := cfg.Mirrors[0]
|
||||
So(m.Name, ShouldEqual, "AOSP")
|
||||
So(m.MirrorDir, ShouldEqual, "/data/git/AOSP")
|
||||
So(m.Provider, ShouldEqual, provCommand)
|
||||
So(m.Interval, ShouldEqual, 720)
|
||||
So(m.Retry, ShouldEqual, 2)
|
||||
So(m.Env["REPO"], ShouldEqual, "/usr/local/bin/aosp-repo")
|
||||
|
||||
m = cfg.Mirrors[1]
|
||||
So(m.Name, ShouldEqual, "debian")
|
||||
So(m.MirrorDir, ShouldEqual, "")
|
||||
So(m.Provider, ShouldEqual, provTwoStageRsync)
|
||||
|
||||
m = cfg.Mirrors[2]
|
||||
So(m.Name, ShouldEqual, "fedora")
|
||||
So(m.MirrorDir, ShouldEqual, "")
|
||||
So(m.Provider, ShouldEqual, provRsync)
|
||||
So(m.ExcludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt")
|
||||
|
||||
m = cfg.Mirrors[3]
|
||||
So(m.Name, ShouldEqual, "debian-security")
|
||||
So(m.MirrorDir, ShouldEqual, "")
|
||||
So(m.Provider, ShouldEqual, provTwoStageRsync)
|
||||
So(m.UseIPv6, ShouldEqual, true)
|
||||
So(m.Stage1Profile, ShouldEqual, "debian")
|
||||
|
||||
m = cfg.Mirrors[4]
|
||||
So(m.Name, ShouldEqual, "ubuntu")
|
||||
So(m.MirrorDir, ShouldEqual, "")
|
||||
So(m.Provider, ShouldEqual, provTwoStageRsync)
|
||||
So(m.UseIPv6, ShouldEqual, true)
|
||||
So(m.Stage1Profile, ShouldEqual, "ubuntu")
|
||||
|
||||
m = cfg.Mirrors[5]
|
||||
So(m.Name, ShouldEqual, "debian-cd")
|
||||
So(m.UseIPv6, ShouldEqual, true)
|
||||
So(m.Provider, ShouldEqual, provRsync)
|
||||
|
||||
So(len(cfg.Mirrors), ShouldEqual, 6)
|
||||
})
|
||||
Convey("Providers can be inited from a valid config file", t, func() {
|
||||
tmpfile, err := ioutil.TempFile("", "tunasync")
|
||||
So(err, ShouldEqual, nil)
|
||||
@ -207,4 +303,90 @@ use_ipv6 = true
|
||||
So(rp.excludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt")
|
||||
|
||||
})
|
||||
|
||||
Convey("MirrorSubdir should work", t, func() {
|
||||
tmpfile, err := ioutil.TempFile("", "tunasync")
|
||||
So(err, ShouldEqual, nil)
|
||||
defer os.Remove(tmpfile.Name())
|
||||
|
||||
cfgBlob1 := `
|
||||
[global]
|
||||
name = "test_worker"
|
||||
log_dir = "/var/log/tunasync/{{.Name}}"
|
||||
mirror_dir = "/data/mirrors"
|
||||
concurrent = 10
|
||||
interval = 240
|
||||
retry = 3
|
||||
|
||||
[manager]
|
||||
api_base = "https://127.0.0.1:5000"
|
||||
token = "some_token"
|
||||
|
||||
[server]
|
||||
hostname = "worker1.example.com"
|
||||
listen_addr = "127.0.0.1"
|
||||
listen_port = 6000
|
||||
ssl_cert = "/etc/tunasync.d/worker1.cert"
|
||||
ssl_key = "/etc/tunasync.d/worker1.key"
|
||||
|
||||
[[mirrors]]
|
||||
name = "ipv6s"
|
||||
use_ipv6 = true
|
||||
[[mirrors.mirrors]]
|
||||
name = "debians"
|
||||
mirror_subdir = "debian"
|
||||
provider = "two-stage-rsync"
|
||||
stage1_profile = "debian"
|
||||
|
||||
[[mirrors.mirrors.mirrors]]
|
||||
name = "debian-security"
|
||||
upstream = "rsync://test.host/debian-security/"
|
||||
[[mirrors.mirrors.mirrors]]
|
||||
name = "ubuntu"
|
||||
stage1_profile = "ubuntu"
|
||||
upstream = "rsync://test.host2/ubuntu/"
|
||||
[[mirrors.mirrors]]
|
||||
name = "debian-cd"
|
||||
provider = "rsync"
|
||||
upstream = "rsync://test.host3/debian-cd/"
|
||||
`
|
||||
err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
|
||||
So(err, ShouldEqual, nil)
|
||||
defer tmpfile.Close()
|
||||
|
||||
cfg, err := LoadConfig(tmpfile.Name())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
providers := map[string]mirrorProvider{}
|
||||
for _, m := range cfg.Mirrors {
|
||||
p := newMirrorProvider(m, cfg)
|
||||
providers[p.Name()] = p
|
||||
}
|
||||
|
||||
p := providers["debian-security"]
|
||||
So(p.Name(), ShouldEqual, "debian-security")
|
||||
So(p.LogDir(), ShouldEqual, "/var/log/tunasync/debian-security")
|
||||
So(p.LogFile(), ShouldEqual, "/var/log/tunasync/debian-security/latest.log")
|
||||
r2p, ok := p.(*twoStageRsyncProvider)
|
||||
So(ok, ShouldBeTrue)
|
||||
So(r2p.stage1Profile, ShouldEqual, "debian")
|
||||
So(r2p.WorkingDir(), ShouldEqual, "/data/mirrors/debian/debian-security")
|
||||
|
||||
p = providers["ubuntu"]
|
||||
So(p.Name(), ShouldEqual, "ubuntu")
|
||||
So(p.LogDir(), ShouldEqual, "/var/log/tunasync/ubuntu")
|
||||
So(p.LogFile(), ShouldEqual, "/var/log/tunasync/ubuntu/latest.log")
|
||||
r2p, ok = p.(*twoStageRsyncProvider)
|
||||
So(ok, ShouldBeTrue)
|
||||
So(r2p.stage1Profile, ShouldEqual, "ubuntu")
|
||||
So(r2p.WorkingDir(), ShouldEqual, "/data/mirrors/debian/ubuntu")
|
||||
|
||||
p = providers["debian-cd"]
|
||||
So(p.Name(), ShouldEqual, "debian-cd")
|
||||
So(p.LogDir(), ShouldEqual, "/var/log/tunasync/debian-cd")
|
||||
So(p.LogFile(), ShouldEqual, "/var/log/tunasync/debian-cd/latest.log")
|
||||
rp, ok := p.(*rsyncProvider)
|
||||
So(ok, ShouldBeTrue)
|
||||
So(rp.WorkingDir(), ShouldEqual, "/data/mirrors/debian-cd")
|
||||
})
|
||||
}
|
||||
|
||||
@ -82,7 +82,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
|
||||
}
|
||||
if mirrorDir == "" {
|
||||
mirrorDir = filepath.Join(
|
||||
cfg.Global.MirrorDir, mirror.Name,
|
||||
cfg.Global.MirrorDir, mirror.MirrorSubDir, mirror.Name,
|
||||
)
|
||||
}
|
||||
if mirror.Interval == 0 {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user