[mirror config] extra rsync options (#107)

* support "rsync_options" array in config

* add test for new options

* fix tests

docs/zh_CN/workers.conf

@@ -45,6 +45,7 @@ docker_image = "tunathu/tunasync-scripts:latest"
 name = "gnu"
 provider = "rsync"
 upstream = "rsync://mirrors.ocf.berkeley.edu/gnu/"
+rsync_options = [ "--delete-excluded" ]
 memory_limit = "256M"
 
 [[mirrors]]
@@ -72,6 +73,7 @@ name = "ubuntu"
 provider = "two-stage-rsync"
 stage1_profile = "debian"
 upstream = "rsync://archive.ubuntu.com/ubuntu/"
+rsync_options = [ "--delete-excluded" ]
 memory_limit = "256M"
 
 # vim: ft=toml

worker/config.go

@@ -129,13 +129,14 @@ type mirrorConfig struct {
 	ExecOnSuccessExtra []string `toml:"exec_on_success_extra"`
 	ExecOnFailureExtra []string `toml:"exec_on_failure_extra"`
 
-	Command       string `toml:"command"`
-	UseIPv6       bool   `toml:"use_ipv6"`
-	UseIPv4       bool   `toml:"use_ipv4"`
-	ExcludeFile   string `toml:"exclude_file"`
-	Username      string `toml:"username"`
-	Password      string `toml:"password"`
-	Stage1Profile string `toml:"stage1_profile"`
+	Command       string   `toml:"command"`
+	UseIPv6       bool     `toml:"use_ipv6"`
+	UseIPv4       bool     `toml:"use_ipv4"`
+	ExcludeFile   string   `toml:"exclude_file"`
+	Username      string   `toml:"username"`
+	Password      string   `toml:"password"`
+	RsyncOptions  []string `toml:"rsync_options"`
+	Stage1Profile string   `toml:"stage1_profile"`
 
 	MemoryLimit string `toml:"memory_limit"`
 

worker/provider.go

@@ -126,19 +126,20 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 		provider = p
 	case provRsync:
 		rc := rsyncConfig{
-			name:        mirror.Name,
-			upstreamURL: mirror.Upstream,
-			rsyncCmd:    mirror.Command,
-			username:    mirror.Username,
-			password:    mirror.Password,
-			excludeFile: mirror.ExcludeFile,
-			workingDir:  mirrorDir,
-			logDir:      logDir,
-			logFile:     filepath.Join(logDir, "latest.log"),
-			useIPv6:     mirror.UseIPv6,
-			useIPv4:     mirror.UseIPv4,
-			interval:    time.Duration(mirror.Interval) * time.Minute,
-			retry:       mirror.Retry,
+			name:         mirror.Name,
+			upstreamURL:  mirror.Upstream,
+			rsyncCmd:     mirror.Command,
+			username:     mirror.Username,
+			password:     mirror.Password,
+			excludeFile:  mirror.ExcludeFile,
+			extraOptions: mirror.RsyncOptions,
+			workingDir:   mirrorDir,
+			logDir:       logDir,
+			logFile:      filepath.Join(logDir, "latest.log"),
+			useIPv6:      mirror.UseIPv6,
+			useIPv4:      mirror.UseIPv4,
+			interval:     time.Duration(mirror.Interval) * time.Minute,
+			retry:        mirror.Retry,
 		}
 		p, err := newRsyncProvider(rc)
 		if err != nil {
@@ -155,6 +156,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 			username:      mirror.Username,
 			password:      mirror.Password,
 			excludeFile:   mirror.ExcludeFile,
+			extraOptions:  mirror.RsyncOptions,
 			workingDir:    mirrorDir,
 			logDir:        logDir,
 			logFile:       filepath.Join(logDir, "latest.log"),

worker/provider_test.go

@@ -116,16 +116,17 @@ func TestRsyncProviderWithAuthentication(t *testing.T) {
 		tmpFile := filepath.Join(tmpDir, "log_file")
 
 		c := rsyncConfig{
-			name:        "tuna",
-			upstreamURL: "rsync://rsync.tuna.moe/tuna/",
-			rsyncCmd:    scriptFile,
-			username:    "tunasync",
-			password:    "tunasyncpassword",
-			workingDir:  tmpDir,
-			logDir:      tmpDir,
-			logFile:     tmpFile,
-			useIPv6:     true,
-			interval:    600 * time.Second,
+			name:         "tuna",
+			upstreamURL:  "rsync://rsync.tuna.moe/tuna/",
+			rsyncCmd:     scriptFile,
+			username:     "tunasync",
+			password:     "tunasyncpassword",
+			workingDir:   tmpDir,
+			extraOptions: []string{"--delete-excluded"},
+			logDir:       tmpDir,
+			logFile:      tmpFile,
+			useIPv4:      true,
+			interval:     600 * time.Second,
 		}
 
 		provider, err := newRsyncProvider(c)
@@ -157,7 +158,7 @@ exit 0
 				fmt.Sprintf(
 					"%s %s -aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
-						"--timeout=120 --contimeout=120 -6 %s %s",
+						"--timeout=120 --contimeout=120 -4 --delete-excluded %s %s",
 					provider.username, provider.password, provider.upstreamURL, provider.WorkingDir(),
 				),
 			)
@@ -319,6 +320,7 @@ func TestTwoStageRsyncProvider(t *testing.T) {
 			logFile:       tmpFile,
 			useIPv6:       true,
 			excludeFile:   tmpFile,
+			extraOptions:  []string{"--delete-excluded", "--cache"},
 			username:      "hello",
 			password:      "world",
 		}
@@ -359,14 +361,14 @@ exit 0
 				fmt.Sprintf(
 					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+
 						"--timeout=120 --contimeout=120 --exclude dists/ -6 "+
-						"--exclude-from %s %s %s",
+						"--exclude-from %s --delete-excluded --cache %s %s",
 					provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 				),
 				targetDir,
 				fmt.Sprintf(
 					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
-						"--timeout=120 --contimeout=120 -6 --exclude-from %s %s %s",
+						"--timeout=120 --contimeout=120 -6 --exclude-from %s --delete-excluded --cache %s %s",
 					provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 				),
 			)
@@ -398,7 +400,7 @@ exit 0
 			expectedOutput := fmt.Sprintf(
 				"-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+
 					"--timeout=120 --contimeout=120 --exclude dists/ -6 "+
-					"--exclude-from %s %s %s\n",
+					"--exclude-from %s --delete-excluded --cache %s %s\n",
 				provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 			)
 

worker/rsync_provider.go

@@ -13,6 +13,7 @@ type rsyncConfig struct {
 	name                                         string
 	rsyncCmd                                     string
 	upstreamURL, username, password, excludeFile string
+	extraOptions                                 []string
 	workingDir, logDir, logFile                  string
 	useIPv6, useIPv4                             bool
 	interval                                     time.Duration
@@ -65,6 +66,9 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 	if c.excludeFile != "" {
 		options = append(options, "--exclude-from", c.excludeFile)
 	}
+	if c.extraOptions != nil {
+		options = append(options, c.extraOptions...)
+	}
 	provider.options = options
 
 	provider.ctx.Set(_WorkingDirKey, c.workingDir)

worker/two_stage_rsync_provider.go

@@ -15,6 +15,7 @@ type twoStageRsyncConfig struct {
 	rsyncCmd                                     string
 	stage1Profile                                string
 	upstreamURL, username, password, excludeFile string
+	extraOptions                                 []string
 	workingDir, logDir, logFile                  string
 	useIPv6                                      bool
 	interval                                     time.Duration
@@ -116,6 +117,9 @@ func (p *twoStageRsyncProvider) Options(stage int) ([]string, error) {
 	if p.excludeFile != "" {
 		options = append(options, "--exclude-from", p.excludeFile)
 	}
+	if p.extraOptions != nil {
+		options = append(options, p.extraOptions...)
+	}
 
 	return options, nil
 }