[mirror config] extra rsync options (#107)

* support "rsync_options" array in config

* add test for new options

* fix tests

docs/zh_CN/workers.conf

@@ -45,6 +45,7 @@ docker_image = "tunathu/tunasync-scripts:latest"
 name = "gnu"
 provider = "rsync"
 upstream = "rsync://mirrors.ocf.berkeley.edu/gnu/"
+rsync_options = [ "--delete-excluded" ]
 memory_limit = "256M"
 
 [[mirrors]]
@@ -72,6 +73,7 @@ name = "ubuntu"
 provider = "two-stage-rsync"
 stage1_profile = "debian"
 upstream = "rsync://archive.ubuntu.com/ubuntu/"
+rsync_options = [ "--delete-excluded" ]
 memory_limit = "256M"
 
 # vim: ft=toml

worker/config.go

@@ -135,6 +135,7 @@ type mirrorConfig struct {
 	ExcludeFile   string   `toml:"exclude_file"`
 	Username      string   `toml:"username"`
 	Password      string   `toml:"password"`
+	RsyncOptions  []string `toml:"rsync_options"`
 	Stage1Profile string   `toml:"stage1_profile"`
 
 	MemoryLimit string `toml:"memory_limit"`

worker/provider.go

@@ -132,6 +132,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 			username:     mirror.Username,
 			password:     mirror.Password,
 			excludeFile:  mirror.ExcludeFile,
+			extraOptions: mirror.RsyncOptions,
 			workingDir:   mirrorDir,
 			logDir:       logDir,
 			logFile:      filepath.Join(logDir, "latest.log"),
@@ -155,6 +156,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 			username:      mirror.Username,
 			password:      mirror.Password,
 			excludeFile:   mirror.ExcludeFile,
+			extraOptions:  mirror.RsyncOptions,
 			workingDir:    mirrorDir,
 			logDir:        logDir,
 			logFile:       filepath.Join(logDir, "latest.log"),

worker/provider_test.go

@@ -122,9 +122,10 @@ func TestRsyncProviderWithAuthentication(t *testing.T) {
 			username:     "tunasync",
 			password:     "tunasyncpassword",
 			workingDir:   tmpDir,
+			extraOptions: []string{"--delete-excluded"},
 			logDir:       tmpDir,
 			logFile:      tmpFile,
-			useIPv6:     true,
+			useIPv4:      true,
 			interval:     600 * time.Second,
 		}
 
@@ -157,7 +158,7 @@ exit 0
 				fmt.Sprintf(
 					"%s %s -aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
-						"--timeout=120 --contimeout=120 -6 %s %s",
+						"--timeout=120 --contimeout=120 -4 --delete-excluded %s %s",
 					provider.username, provider.password, provider.upstreamURL, provider.WorkingDir(),
 				),
 			)
@@ -319,6 +320,7 @@ func TestTwoStageRsyncProvider(t *testing.T) {
 			logFile:       tmpFile,
 			useIPv6:       true,
 			excludeFile:   tmpFile,
+			extraOptions:  []string{"--delete-excluded", "--cache"},
 			username:      "hello",
 			password:      "world",
 		}
@@ -359,14 +361,14 @@ exit 0
 				fmt.Sprintf(
 					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+
 						"--timeout=120 --contimeout=120 --exclude dists/ -6 "+
-						"--exclude-from %s %s %s",
+						"--exclude-from %s --delete-excluded --cache %s %s",
 					provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 				),
 				targetDir,
 				fmt.Sprintf(
 					"-aHvh --no-o --no-g --stats --exclude .~tmp~/ "+
 						"--delete --delete-after --delay-updates --safe-links "+
-						"--timeout=120 --contimeout=120 -6 --exclude-from %s %s %s",
+						"--timeout=120 --contimeout=120 -6 --exclude-from %s --delete-excluded --cache %s %s",
 					provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 				),
 			)
@@ -398,7 +400,7 @@ exit 0
 			expectedOutput := fmt.Sprintf(
 				"-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+
 					"--timeout=120 --contimeout=120 --exclude dists/ -6 "+
-					"--exclude-from %s %s %s\n",
+					"--exclude-from %s --delete-excluded --cache %s %s\n",
 				provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
 			)
 

worker/rsync_provider.go

@@ -13,6 +13,7 @@ type rsyncConfig struct {
 	name                                         string
 	rsyncCmd                                     string
 	upstreamURL, username, password, excludeFile string
+	extraOptions                                 []string
 	workingDir, logDir, logFile                  string
 	useIPv6, useIPv4                             bool
 	interval                                     time.Duration
@@ -65,6 +66,9 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 	if c.excludeFile != "" {
 		options = append(options, "--exclude-from", c.excludeFile)
 	}
+	if c.extraOptions != nil {
+		options = append(options, c.extraOptions...)
+	}
 	provider.options = options
 
 	provider.ctx.Set(_WorkingDirKey, c.workingDir)

worker/two_stage_rsync_provider.go

@@ -15,6 +15,7 @@ type twoStageRsyncConfig struct {
 	rsyncCmd                                     string
 	stage1Profile                                string
 	upstreamURL, username, password, excludeFile string
+	extraOptions                                 []string
 	workingDir, logDir, logFile                  string
 	useIPv6                                      bool
 	interval                                     time.Duration
@@ -116,6 +117,9 @@ func (p *twoStageRsyncProvider) Options(stage int) ([]string, error) {
 	if p.excludeFile != "" {
 		options = append(options, "--exclude-from", p.excludeFile)
 	}
+	if p.extraOptions != nil {
+		options = append(options, p.extraOptions...)
+	}
 
 	return options, nil
 }