Bump to v0.9.3

Signed-off-by: Shengqi Chen <harry-chen@outlook.com>

.github/workflows/release.yml

@@ -22,9 +22,10 @@ jobs:
       id: go
     - name: Build
       run: |
+        TAG=$(git describe --tags)
         for i in linux-amd64 linux-arm64 linux-riscv64 linux-loong64; do
           make ARCH=$i all
-          tar -cz --numeric-owner --owner root --group root -f tunasync-$i-bin.tar.gz -C build-$i tunasync tunasynctl
+          tar -cz --numeric-owner --owner root --group root -f tunasync-${TAG}-$i-bin.tar.gz -C build-$i tunasync tunasynctl
         done
     - name: Create Release
       uses: softprops/action-gh-release@v2

cmd/tunasync/tunasync.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/moby/moby/pkg/reexec"
+	"github.com/moby/sys/reexec"
 	"github.com/pkg/profile"
 	"github.com/urfave/cli"
 	"gopkg.in/op/go-logging.v1"

go.mod

@@ -18,7 +18,8 @@ require (
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/imdario/mergo v0.3.16
 	github.com/moby/moby v28.0.1+incompatible
-	github.com/opencontainers/runtime-spec v1.2.0
+	github.com/moby/sys/reexec v0.1.0
+	github.com/opencontainers/runtime-spec v1.2.1
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/profile v1.7.0
 	github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46
@@ -67,7 +68,6 @@ require (
 	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/moby/sys/reexec v0.1.0 // indirect
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

go.sum

@@ -212,6 +212,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
 github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
 github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk=
 github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww=
+github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=

internal/version.go

@@ -1,4 +1,4 @@
 package internal
 
 // Version of the program
-const Version string = "0.9.2"
+const Version string = "0.9.3"

worker/base_provider.go

@@ -19,9 +19,10 @@ type baseProvider struct {
 	timeout  time.Duration
 	isMaster bool
 
-	cmd       *cmdJob
+	cmd              *cmdJob
-	logFileFd *os.File
+	logFileFd        *os.File
-	isRunning atomic.Value
+	isRunning        atomic.Value
+	successExitCodes []int
 
 	cgroup *cgroupHook
 	zfs    *zfsHook
@@ -186,3 +187,18 @@ func (p *baseProvider) Terminate() error {
 func (p *baseProvider) DataSize() string {
 	return ""
 }
+
+func (p *baseProvider) SetSuccessExitCodes(codes []int) {
+	if codes == nil {
+		p.successExitCodes = []int{}
+	} else {
+		p.successExitCodes = codes
+	}
+}
+
+func (p *baseProvider) GetSuccessExitCodes() []int {
+	if p.successExitCodes == nil {
+		return []int{}
+	}
+	return p.successExitCodes
+}

worker/cgroup.go

@@ -15,7 +15,7 @@ import (
 	cgroups "github.com/containerd/cgroups/v3"
 	cgv1 "github.com/containerd/cgroups/v3/cgroup1"
 	cgv2 "github.com/containerd/cgroups/v3/cgroup2"
-	"github.com/moby/moby/pkg/reexec"
+	"github.com/moby/sys/reexec"
 	contspecs "github.com/opencontainers/runtime-spec/specs-go"
 )
 

worker/cgroup_test.go

@@ -14,7 +14,7 @@ import (
 	cgv1 "github.com/containerd/cgroups/v3/cgroup1"
 	cgv2 "github.com/containerd/cgroups/v3/cgroup2"
 	units "github.com/docker/go-units"
-	"github.com/moby/moby/pkg/reexec"
+	"github.com/moby/sys/reexec"
 
 	. "github.com/smartystreets/goconvey/convey"
 )
@@ -250,7 +250,7 @@ sleep 30
 				if cgcf.Group == "" {
 					wkrg, err := cgv2.NestedGroupPath("")
 					So(err, ShouldBeNil)
-					wkrMgr, err := cgv2.Load(wkrg)
+					wkrMgr, _ := cgv2.Load(wkrg)
 					allCtrls, err := wkrMgr.Controllers()
 					So(err, ShouldBeNil)
 					err = wkrMgr.ToggleControllers(allCtrls, cgv2.Disable)

worker/config.go

@@ -63,6 +63,9 @@ type globalConfig struct {
 
 	ExecOnSuccess []string `toml:"exec_on_success"`
 	ExecOnFailure []string `toml:"exec_on_failure"`
+
+	// merged with mirror-specific options. make sure you know what you are doing!
+	SuccessExitCodes []int `toml:"dangerous_global_success_exit_codes"`
 }
 
 type managerConfig struct {
@@ -169,6 +172,9 @@ type mirrorConfig struct {
 	ExecOnSuccessExtra []string `toml:"exec_on_success_extra"`
 	ExecOnFailureExtra []string `toml:"exec_on_failure_extra"`
 
+	// will be merged with global option
+	SuccessExitCodes []int `toml:"success_exit_codes"`
+
 	Command           string   `toml:"command"`
 	FailOnMatch       string   `toml:"fail_on_match"`
 	SizePattern       string   `toml:"size_pattern"`

worker/config_diff_test.go

@@ -10,12 +10,12 @@ import (
 func TestConfigDiff(t *testing.T) {
 	Convey("When old and new configs are equal", t, func() {
 		oldList := []mirrorConfig{
-			mirrorConfig{Name: "debian"},
+			{Name: "debian"},
-			mirrorConfig{Name: "debian-security"},
+			{Name: "debian-security"},
-			mirrorConfig{Name: "fedora"},
+			{Name: "fedora"},
-			mirrorConfig{Name: "archlinux"},
+			{Name: "archlinux"},
-			mirrorConfig{Name: "AOSP"},
+			{Name: "AOSP"},
-			mirrorConfig{Name: "ubuntu"},
+			{Name: "ubuntu"},
 		}
 		newList := make([]mirrorConfig, len(oldList))
 		copy(newList, oldList)
@@ -25,12 +25,12 @@ func TestConfigDiff(t *testing.T) {
 	})
 	Convey("When old config is empty", t, func() {
 		newList := []mirrorConfig{
-			mirrorConfig{Name: "debian"},
+			{Name: "debian"},
-			mirrorConfig{Name: "debian-security"},
+			{Name: "debian-security"},
-			mirrorConfig{Name: "fedora"},
+			{Name: "fedora"},
-			mirrorConfig{Name: "archlinux"},
+			{Name: "archlinux"},
-			mirrorConfig{Name: "AOSP"},
+			{Name: "AOSP"},
-			mirrorConfig{Name: "ubuntu"},
+			{Name: "ubuntu"},
 		}
 		oldList := make([]mirrorConfig, 0)
 
@@ -39,12 +39,12 @@ func TestConfigDiff(t *testing.T) {
 	})
 	Convey("When new config is empty", t, func() {
 		oldList := []mirrorConfig{
-			mirrorConfig{Name: "debian"},
+			{Name: "debian"},
-			mirrorConfig{Name: "debian-security"},
+			{Name: "debian-security"},
-			mirrorConfig{Name: "fedora"},
+			{Name: "fedora"},
-			mirrorConfig{Name: "archlinux"},
+			{Name: "archlinux"},
-			mirrorConfig{Name: "AOSP"},
+			{Name: "AOSP"},
-			mirrorConfig{Name: "ubuntu"},
+			{Name: "ubuntu"},
 		}
 		newList := make([]mirrorConfig, 0)
 
@@ -53,19 +53,19 @@ func TestConfigDiff(t *testing.T) {
 	})
 	Convey("When giving two config lists with different names", t, func() {
 		oldList := []mirrorConfig{
-			mirrorConfig{Name: "debian"},
+			{Name: "debian"},
-			mirrorConfig{Name: "debian-security"},
+			{Name: "debian-security"},
-			mirrorConfig{Name: "fedora"},
+			{Name: "fedora"},
-			mirrorConfig{Name: "archlinux"},
+			{Name: "archlinux"},
-			mirrorConfig{Name: "AOSP", Env: map[string]string{"REPO": "/usr/bin/repo"}},
+			{Name: "AOSP", Env: map[string]string{"REPO": "/usr/bin/repo"}},
-			mirrorConfig{Name: "ubuntu"},
+			{Name: "ubuntu"},
 		}
 		newList := []mirrorConfig{
-			mirrorConfig{Name: "debian"},
+			{Name: "debian"},
-			mirrorConfig{Name: "debian-cd"},
+			{Name: "debian-cd"},
-			mirrorConfig{Name: "archlinuxcn"},
+			{Name: "archlinuxcn"},
-			mirrorConfig{Name: "AOSP", Env: map[string]string{"REPO": "/usr/local/bin/aosp-repo"}},
+			{Name: "AOSP", Env: map[string]string{"REPO": "/usr/local/bin/aosp-repo"}},
-			mirrorConfig{Name: "ubuntu-ports"},
+			{Name: "ubuntu-ports"},
 		}
 
 		difference := diffMirrorConfig(oldList, newList)

worker/config_test.go

@@ -521,4 +521,60 @@ rsync_options = ["--local"]
 			"--local",       // from mirror.rsync_options
 		})
 	})
+
+	Convey("success_exit_codes should work globally and per mirror", t, func() {
+		tmpfile, err := os.CreateTemp("", "tunasync")
+		So(err, ShouldEqual, nil)
+		defer os.Remove(tmpfile.Name())
+
+		cfgBlob1 := `
+[global]
+name = "test_worker"
+log_dir = "/var/log/tunasync/{{.Name}}"
+mirror_dir = "/data/mirrors"
+concurrent = 10
+interval = 240
+retry = 3
+timeout = 86400
+dangerous_global_success_exit_codes = [10, 20]
+
+[manager]
+api_base = "https://127.0.0.1:5000"
+token = "some_token"
+
+[server]
+hostname = "worker1.example.com"
+listen_addr = "127.0.0.1"
+listen_port = 6000
+ssl_cert = "/etc/tunasync.d/worker1.cert"
+ssl_key = "/etc/tunasync.d/worker1.key"
+
+[[mirrors]]
+name = "foo"
+provider = "rsync"
+upstream = "rsync://foo.bar/"
+interval = 720
+retry = 2
+timeout = 3600
+mirror_dir = "/data/foo"
+success_exit_codes = [30, 40]
+`
+
+		err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
+		So(err, ShouldEqual, nil)
+		defer tmpfile.Close()
+
+		cfg, err := LoadConfig(tmpfile.Name())
+		So(err, ShouldBeNil)
+
+		providers := map[string]mirrorProvider{}
+		for _, m := range cfg.Mirrors {
+			p := newMirrorProvider(m, cfg)
+			providers[p.Name()] = p
+		}
+
+		p, ok := providers["foo"].(*rsyncProvider)
+		So(ok, ShouldBeTrue)
+		So(p.successExitCodes, ShouldResemble, []int{10, 20, 30, 40})
+	})
 }

worker/provider.go

@@ -60,6 +60,10 @@ type mirrorProvider interface {
 	ExitContext() *Context
 	// return context
 	Context() *Context
+
+	// set in newMirrorProvider, used by cmdJob.Wait
+	SetSuccessExitCodes(codes []int)
+	GetSuccessExitCodes() []int
 }
 
 // newProvider creates a mirrorProvider instance
@@ -249,5 +253,17 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 	}
 	addHookFromCmdList(mirror.ExecOnFailureExtra, execOnFailure)
 
+	successExitCodes := []int{}
+	if cfg.Global.SuccessExitCodes != nil {
+		successExitCodes = append(successExitCodes, cfg.Global.SuccessExitCodes...)
+	}
+	if mirror.SuccessExitCodes != nil {
+		successExitCodes = append(successExitCodes, mirror.SuccessExitCodes...)
+	}
+	if len(successExitCodes) > 0 {
+		logger.Infof("Non-zero success exit codes set for mirror %s: %v", mirror.Name, successExitCodes)
+		provider.SetSuccessExitCodes(successExitCodes)
+	}
+
 	return provider
 }

worker/provider_test.go

@@ -552,6 +552,59 @@ sleep 10
 			So(provider.DataSize(), ShouldBeEmpty)
 		})
 	})
+	Convey("Command Provider with successExitCodes should work", t, func(ctx C) {
+		tmpDir, err := os.MkdirTemp("", "tunasync")
+		defer os.RemoveAll(tmpDir)
+		So(err, ShouldBeNil)
+		scriptFile := filepath.Join(tmpDir, "cmd.sh")
+		tmpFile := filepath.Join(tmpDir, "log_file")
+
+		c := cmdConfig{
+			name:        "tuna-cmd",
+			upstreamURL: "http://mirrors.tuna.moe/",
+			command:     "bash " + scriptFile,
+			workingDir:  tmpDir,
+			logDir:      tmpDir,
+			logFile:     tmpFile,
+			interval:    600 * time.Second,
+		}
+
+		provider, err := newCmdProvider(c)
+		provider.SetSuccessExitCodes([]int{199, 200})
+		So(err, ShouldBeNil)
+
+		So(provider.Type(), ShouldEqual, provCommand)
+		So(provider.Name(), ShouldEqual, c.name)
+		So(provider.WorkingDir(), ShouldEqual, c.workingDir)
+		So(provider.LogDir(), ShouldEqual, c.logDir)
+		So(provider.LogFile(), ShouldEqual, c.logFile)
+		So(provider.Interval(), ShouldEqual, c.interval)
+		So(provider.GetSuccessExitCodes(), ShouldResemble, []int{199, 200})
+
+		Convey("Command exits with configured successExitCodes", func() {
+			scriptContent := `exit 199`
+			err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
+			So(err, ShouldBeNil)
+			readedScriptContent, err := os.ReadFile(scriptFile)
+			So(err, ShouldBeNil)
+			So(readedScriptContent, ShouldResemble, []byte(scriptContent))
+
+			err = provider.Run(make(chan empty, 1))
+			So(err, ShouldBeNil)
+		})
+
+		Convey("Command exits with unknown exit code", func() {
+			scriptContent := `exit 201`
+			err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
+			So(err, ShouldBeNil)
+			readedScriptContent, err := os.ReadFile(scriptFile)
+			So(err, ShouldBeNil)
+			So(readedScriptContent, ShouldResemble, []byte(scriptContent))
+
+			err = provider.Run(make(chan empty, 1))
+			So(err, ShouldNotBeNil)
+		})
+	})
 }
 
 func TestTwoStageRsyncProvider(t *testing.T) {

worker/rsync_provider.go

@@ -77,7 +77,7 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 		options = c.overriddenOptions
 	}
 
-	if c.useOverrideOnly == true {
+	if c.useOverrideOnly {
 		if c.overriddenOptions == nil {
 			return nil, errors.New("rsync_override_only is set but no rsync_override provided")
 		}

worker/runner.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"slices"
 	"strings"
 	"sync"
 	"syscall"
@@ -12,7 +13,7 @@ import (
 
 	"github.com/codeskyblue/go-sh"
 	cgv1 "github.com/containerd/cgroups/v3/cgroup1"
-	"github.com/moby/moby/pkg/reexec"
+	"github.com/moby/sys/reexec"
 	"golang.org/x/sys/unix"
 )
 
@@ -171,9 +172,18 @@ func (c *cmdJob) Wait() error {
 		return c.retErr
 	default:
 		err := c.cmd.Wait()
-		c.retErr = err
 		close(c.finished)
-		return err
+		if err != nil {
+			code := err.(*exec.ExitError).ExitCode()
+			allowedCodes := c.provider.GetSuccessExitCodes()
+			if slices.Contains(allowedCodes, code) {
+				// process exited with non-success status
+				logger.Infof("Command %s exited with code %d: treated as success (allowed: %v)", c.cmd.Args, code, allowedCodes)
+			} else {
+				c.retErr = err
+			}
+		}
+		return c.retErr
 	}
 }
 

worker/worker_test.go

@@ -147,7 +147,7 @@ func TestWorker(t *testing.T) {
 		})
 		Convey("with one job", func(ctx C) {
 			workerCfg.Mirrors = []mirrorConfig{
-				mirrorConfig{
+				{
 					Name:     "job-ls",
 					Provider: provCommand,
 					Command:  "ls",
@@ -194,17 +194,17 @@ func TestWorker(t *testing.T) {
 		})
 		Convey("with several jobs", func(ctx C) {
 			workerCfg.Mirrors = []mirrorConfig{
-				mirrorConfig{
+				{
 					Name:     "job-ls-1",
 					Provider: provCommand,
 					Command:  "ls",
 				},
-				mirrorConfig{
+				{
 					Name:     "job-fail",
 					Provider: provCommand,
 					Command:  "non-existent-command-xxxx",
 				},
-				mirrorConfig{
+				{
 					Name:     "job-ls-2",
 					Provider: provCommand,
 					Command:  "ls",

worker/zfs_hook_test.go

@@ -12,7 +12,7 @@ import (
 func TestZFSHook(t *testing.T) {
 
 	Convey("ZFS Hook should work", t, func(ctx C) {
-		tmpDir, err := os.MkdirTemp("", "tunasync")
+		tmpDir, _ := os.MkdirTemp("", "tunasync")
 		tmpFile := filepath.Join(tmpDir, "log_file")
 
 		c := cmdConfig{