Merge pull request #208 from tuna/rsync-opts

More configuration options for rsync

worker/config.go

@@ -58,6 +58,9 @@ type globalConfig struct {
 	Retry      int    `toml:"retry"`
 	Timeout    int    `toml:"timeout"`
 
+	// appended to the options generated by rsync_provider, but before mirror-specific options
+	RsyncOptions []string `toml:"rsync_options"`
+
 	ExecOnSuccess []string `toml:"exec_on_success"`
 	ExecOnFailure []string `toml:"exec_on_failure"`
 }
@@ -162,7 +165,7 @@ type mirrorConfig struct {
 	ExecOnSuccess []string `toml:"exec_on_success"`
 	ExecOnFailure []string `toml:"exec_on_failure"`
 
-	// These two options  the global options
+	// These two options are appended to the global options
 	ExecOnSuccessExtra []string `toml:"exec_on_success_extra"`
 	ExecOnFailureExtra []string `toml:"exec_on_failure_extra"`
 
@@ -178,6 +181,7 @@ type mirrorConfig struct {
 	RsyncTimeout      int      `toml:"rsync_timeout"`
 	RsyncOptions      []string `toml:"rsync_options"`
 	RsyncOverride     []string `toml:"rsync_override"`
+	RsyncOverrideOnly bool     `toml:"rsync_override_only"` // only use provided overridden options if true
 	Stage1Profile     string   `toml:"stage1_profile"`
 
 	MemoryLimit MemBytes `toml:"memory_limit"`

worker/config_test.go

@@ -403,4 +403,122 @@ use_ipv6 = true
 		So(rp.WorkingDir(), ShouldEqual, "/data/mirrors/debian-cd")
 		So(p.Timeout(), ShouldEqual, 86400*time.Second)
 	})
+
+	Convey("rsync_override_only should work", t, func() {
+		tmpfile, err := os.CreateTemp("", "tunasync")
+		So(err, ShouldEqual, nil)
+		defer os.Remove(tmpfile.Name())
+
+		cfgBlob1 := `
+[global]
+name = "test_worker"
+log_dir = "/var/log/tunasync/{{.Name}}"
+mirror_dir = "/data/mirrors"
+concurrent = 10
+interval = 240
+retry = 3
+timeout = 86400
+
+[manager]
+api_base = "https://127.0.0.1:5000"
+token = "some_token"
+
+[server]
+hostname = "worker1.example.com"
+listen_addr = "127.0.0.1"
+listen_port = 6000
+ssl_cert = "/etc/tunasync.d/worker1.cert"
+ssl_key = "/etc/tunasync.d/worker1.key"
+
+[[mirrors]]
+name = "foo"
+provider = "rsync"
+upstream = "rsync://foo.bar/"
+interval = 720
+retry = 2
+timeout = 3600
+mirror_dir = "/data/foo"
+rsync_override = ["--bar", "baz"]
+rsync_override_only = true
+`
+
+		err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
+		So(err, ShouldEqual, nil)
+		defer tmpfile.Close()
+
+		cfg, err := LoadConfig(tmpfile.Name())
+		So(err, ShouldBeNil)
+
+		providers := map[string]mirrorProvider{}
+		for _, m := range cfg.Mirrors {
+			p := newMirrorProvider(m, cfg)
+			providers[p.Name()] = p
+		}
+
+		p, ok := providers["foo"].(*rsyncProvider)
+		So(ok, ShouldBeTrue)
+		So(p.options, ShouldResemble, []string{"--bar", "baz"})
+	})
+
+	Convey("rsync global options should work", t, func() {
+		tmpfile, err := os.CreateTemp("", "tunasync")
+		So(err, ShouldEqual, nil)
+		defer os.Remove(tmpfile.Name())
+
+		cfgBlob1 := `
+[global]
+name = "test_worker"
+log_dir = "/var/log/tunasync/{{.Name}}"
+mirror_dir = "/data/mirrors"
+concurrent = 10
+interval = 240
+retry = 3
+timeout = 86400
+rsync_options = ["--global"]
+
+[manager]
+api_base = "https://127.0.0.1:5000"
+token = "some_token"
+
+[server]
+hostname = "worker1.example.com"
+listen_addr = "127.0.0.1"
+listen_port = 6000
+ssl_cert = "/etc/tunasync.d/worker1.cert"
+ssl_key = "/etc/tunasync.d/worker1.key"
+
+[[mirrors]]
+name = "foo"
+provider = "rsync"
+upstream = "rsync://foo.bar/"
+interval = 720
+retry = 2
+timeout = 3600
+mirror_dir = "/data/foo"
+rsync_override = ["--override"]
+rsync_options = ["--local"]
+`
+
+		err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
+		So(err, ShouldEqual, nil)
+		defer tmpfile.Close()
+
+		cfg, err := LoadConfig(tmpfile.Name())
+		So(err, ShouldBeNil)
+
+		providers := map[string]mirrorProvider{}
+		for _, m := range cfg.Mirrors {
+			p := newMirrorProvider(m, cfg)
+			providers[p.Name()] = p
+		}
+
+		p, ok := providers["foo"].(*rsyncProvider)
+		So(ok, ShouldBeTrue)
+		So(p.options, ShouldResemble, []string{
+			"--override",    // from mirror.rsync_override
+			"--timeout=120", // generated by newRsyncProvider
+			"--global",      // from global.rsync_options
+			"--local",       // from mirror.rsync_options
+		})
+	})
 }

worker/provider.go

@@ -142,7 +142,9 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 			extraOptions:      mirror.RsyncOptions,
 			rsyncNeverTimeout: mirror.RsyncNoTimeo,
 			rsyncTimeoutValue: mirror.RsyncTimeout,
+			globalOptions:     cfg.Global.RsyncOptions,
 			overriddenOptions: mirror.RsyncOverride,
+			useOverrideOnly:   mirror.RsyncOverrideOnly,
 			rsyncEnv:          mirror.Env,
 			workingDir:        mirrorDir,
 			logDir:            logDir,

worker/rsync_provider.go

@@ -14,7 +14,9 @@ type rsyncConfig struct {
 	rsyncCmd                                     string
 	upstreamURL, username, password, excludeFile string
 	extraOptions                                 []string
+	globalOptions                                []string
 	overriddenOptions                            []string
+	useOverrideOnly                              bool
 	rsyncNeverTimeout                            bool
 	rsyncTimeoutValue                            int
 	rsyncEnv                                     map[string]string
@@ -75,6 +77,12 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 		options = c.overriddenOptions
 	}
 
+	if c.useOverrideOnly == true {
+		if c.overriddenOptions == nil {
+			return nil, errors.New("rsync_override_only is set but no rsync_override provided")
+		}
+		// use overridden options only
+	} else {
 		if !c.rsyncNeverTimeout {
 			timeo := 120
 			if c.rsyncTimeoutValue > 0 {
@@ -92,9 +100,14 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 		if c.excludeFile != "" {
 			options = append(options, "--exclude-from", c.excludeFile)
 		}
+
+		if c.globalOptions != nil {
+			options = append(options, c.globalOptions...)
+		}
 		if c.extraOptions != nil {
 			options = append(options, c.extraOptions...)
 		}
+	}
 	provider.options = options
 
 	provider.ctx.Set(_WorkingDirKey, c.workingDir)