Implement rsync_override_only in mirrorConfig

Signed-off-by: Harry Chen <i@harrychen.xyz>

worker/config.go

@@ -166,19 +166,20 @@ type mirrorConfig struct {
 	ExecOnSuccessExtra []string `toml:"exec_on_success_extra"`
 	ExecOnFailureExtra []string `toml:"exec_on_failure_extra"`
 
-	Command       string   `toml:"command"`
+	Command           string   `toml:"command"`
-	FailOnMatch   string   `toml:"fail_on_match"`
+	FailOnMatch       string   `toml:"fail_on_match"`
-	SizePattern   string   `toml:"size_pattern"`
+	SizePattern       string   `toml:"size_pattern"`
-	UseIPv6       bool     `toml:"use_ipv6"`
+	UseIPv6           bool     `toml:"use_ipv6"`
-	UseIPv4       bool     `toml:"use_ipv4"`
+	UseIPv4           bool     `toml:"use_ipv4"`
-	ExcludeFile   string   `toml:"exclude_file"`
+	ExcludeFile       string   `toml:"exclude_file"`
-	Username      string   `toml:"username"`
+	Username          string   `toml:"username"`
-	Password      string   `toml:"password"`
+	Password          string   `toml:"password"`
-	RsyncNoTimeo  bool     `toml:"rsync_no_timeout"`
+	RsyncNoTimeo      bool     `toml:"rsync_no_timeout"`
-	RsyncTimeout  int      `toml:"rsync_timeout"`
+	RsyncTimeout      int      `toml:"rsync_timeout"`
-	RsyncOptions  []string `toml:"rsync_options"`
+	RsyncOptions      []string `toml:"rsync_options"`
-	RsyncOverride []string `toml:"rsync_override"`
+	RsyncOverride     []string `toml:"rsync_override"`
-	Stage1Profile string   `toml:"stage1_profile"`
+	RsyncOverrideOnly bool     `toml:"rsync_override_only"` // only use provided overridden options if true
+	Stage1Profile     string   `toml:"stage1_profile"`
 
 	MemoryLimit MemBytes `toml:"memory_limit"`
 

worker/config_test.go

@@ -403,4 +403,60 @@ use_ipv6 = true
 		So(rp.WorkingDir(), ShouldEqual, "/data/mirrors/debian-cd")
 		So(p.Timeout(), ShouldEqual, 86400*time.Second)
 	})
+
+	Convey("rsync_override_only should work", t, func() {
+		tmpfile, err := os.CreateTemp("", "tunasync")
+		So(err, ShouldEqual, nil)
+		defer os.Remove(tmpfile.Name())
+
+		cfgBlob1 := `
+[global]
+name = "test_worker"
+log_dir = "/var/log/tunasync/{{.Name}}"
+mirror_dir = "/data/mirrors"
+concurrent = 10
+interval = 240
+retry = 3
+timeout = 86400
+
+[manager]
+api_base = "https://127.0.0.1:5000"
+token = "some_token"
+
+[server]
+hostname = "worker1.example.com"
+listen_addr = "127.0.0.1"
+listen_port = 6000
+ssl_cert = "/etc/tunasync.d/worker1.cert"
+ssl_key = "/etc/tunasync.d/worker1.key"
+
+[[mirrors]]
+name = "foo"
+provider = "rsync"
+upstream = "rsync://foo.bar/"
+interval = 720
+retry = 2
+timeout = 3600
+mirror_dir = "/data/foo"
+rsync_override = ["--bar", "baz"]
+rsync_override_only = true
+`
+
+		err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
+		So(err, ShouldEqual, nil)
+		defer tmpfile.Close()
+
+		cfg, err := LoadConfig(tmpfile.Name())
+		So(err, ShouldBeNil)
+
+		providers := map[string]mirrorProvider{}
+		for _, m := range cfg.Mirrors {
+			p := newMirrorProvider(m, cfg)
+			providers[p.Name()] = p
+		}
+
+		p, ok := providers["foo"].(*rsyncProvider)
+		So(ok, ShouldBeTrue)
+		So(p.options, ShouldResemble, []string{"--bar", "baz"})
+	})
 }

worker/provider.go

@@ -143,6 +143,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 			rsyncNeverTimeout: mirror.RsyncNoTimeo,
 			rsyncTimeoutValue: mirror.RsyncTimeout,
 			overriddenOptions: mirror.RsyncOverride,
+			useOverrideOnly:   mirror.RsyncOverrideOnly,
 			rsyncEnv:          mirror.Env,
 			workingDir:        mirrorDir,
 			logDir:            logDir,

worker/rsync_provider.go

@@ -15,6 +15,7 @@ type rsyncConfig struct {
 	upstreamURL, username, password, excludeFile string
 	extraOptions                                 []string
 	overriddenOptions                            []string
+	useOverrideOnly                              bool
 	rsyncNeverTimeout                            bool
 	rsyncTimeoutValue                            int
 	rsyncEnv                                     map[string]string
@@ -75,25 +76,31 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 		options = c.overriddenOptions
 	}
 
-	if !c.rsyncNeverTimeout {
+	if c.useOverrideOnly == true {
-		timeo := 120
+		if c.overriddenOptions == nil {
-		if c.rsyncTimeoutValue > 0 {
+			return nil, errors.New("rsync_override_only is set but no rsync_override provided")
-			timeo = c.rsyncTimeoutValue
+		}
+	} else {
+		if !c.rsyncNeverTimeout {
+			timeo := 120
+			if c.rsyncTimeoutValue > 0 {
+				timeo = c.rsyncTimeoutValue
+			}
+			options = append(options, fmt.Sprintf("--timeout=%d", timeo))
 		}
-		options = append(options, fmt.Sprintf("--timeout=%d", timeo))
-	}
 
-	if c.useIPv6 {
+		if c.useIPv6 {
-		options = append(options, "-6")
+			options = append(options, "-6")
-	} else if c.useIPv4 {
+		} else if c.useIPv4 {
-		options = append(options, "-4")
+			options = append(options, "-4")
-	}
+		}
 
-	if c.excludeFile != "" {
+		if c.excludeFile != "" {
-		options = append(options, "--exclude-from", c.excludeFile)
+			options = append(options, "--exclude-from", c.excludeFile)
-	}
+		}
-	if c.extraOptions != nil {
+		if c.extraOptions != nil {
-		options = append(options, c.extraOptions...)
+			options = append(options, c.extraOptions...)
+		}
 	}
 	provider.options = options