yacloud/tunasync
1
0
Fork 0
mirror of https://github.com/tuna/tunasync.git synced 2025-04-20 11:42:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix more linter issues

Browse Source 
Signed-off-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
Shengqi Chen 2025-01-11 15:53:42 +08:00
parent ab416f6545
commit 99c7ab6b65
No known key found for this signature in database
19 changed files with 378 additions and 386 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/tunasynctl/tunasynctl.go
View File

@ -3,7 +3,7 @@ package main
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil" "io"
"net/http" "net/http"
"os" "os"
"strconv" "strconv"
@ -122,7 +122,7 @@ func initialize(c *cli.Context) error {
var err error var err error
client, err = tunasync.CreateHTTPClient(cfg.CACert) client, err = tunasync.CreateHTTPClient(cfg.CACert)
if err != nil { if err != nil {
err = fmt.Errorf("Error initializing HTTP client: %s", err.Error()) err = fmt.Errorf("error initializing HTTP client: %s", err.Error())
// logger.Error(err.Error()) // logger.Error(err.Error())
return err return err
@ -292,7 +292,7 @@ func updateMirrorSize(c *cli.Context) error {
1) 1)
} }
defer resp.Body.Close() defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body) body, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
return cli.NewExitError( return cli.NewExitError(
fmt.Sprintf("Manager failed to update mirror size: %s", body), 1, fmt.Sprintf("Manager failed to update mirror size: %s", body), 1,
@ -338,7 +338,7 @@ func removeWorker(c *cli.Context) error {
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return cli.NewExitError( return cli.NewExitError(
fmt.Sprintf("Failed to parse response: %s", err.Error()), fmt.Sprintf("Failed to parse response: %s", err.Error()),
@ -351,7 +351,7 @@ func removeWorker(c *cli.Context) error {
} }
res := map[string]string{} res := map[string]string{}
err = json.NewDecoder(resp.Body).Decode(&res) _ = json.NewDecoder(resp.Body).Decode(&res)
if res["message"] == "deleted" { if res["message"] == "deleted" {
fmt.Println("Successfully removed the worker") fmt.Println("Successfully removed the worker")
} else { } else {
@ -376,7 +376,7 @@ func flushDisabledJobs(c *cli.Context) error {
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return cli.NewExitError( return cli.NewExitError(
fmt.Sprintf("Failed to parse response: %s", err.Error()), fmt.Sprintf("Failed to parse response: %s", err.Error()),
@ -430,7 +430,7 @@ func cmdJob(cmd tunasync.CmdVerb) cli.ActionFunc {
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return cli.NewExitError( return cli.NewExitError(
fmt.Sprintf("Failed to parse response: %s", err.Error()), fmt.Sprintf("Failed to parse response: %s", err.Error()),
@ -468,7 +468,7 @@ func cmdWorker(cmd tunasync.CmdVerb) cli.ActionFunc {
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
body, err := ioutil.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return cli.NewExitError( return cli.NewExitError(
fmt.Sprintf("Failed to parse response: %s", err.Error()), fmt.Sprintf("Failed to parse response: %s", err.Error()),

17
internal/util.go
View File

@ -7,8 +7,9 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io/ioutil" "io"
"net/http" "net/http"
"os"
"os/exec" "os/exec"
"regexp" "regexp"
"time" "time"
@ -39,19 +40,19 @@ var rsyncExitValues = map[int]string{
// GetTLSConfig generate tls.Config from CAFile // GetTLSConfig generate tls.Config from CAFile
func GetTLSConfig(CAFile string) (*tls.Config, error) { func GetTLSConfig(CAFile string) (*tls.Config, error) {
caCert, err := ioutil.ReadFile(CAFile) caCert, err := os.ReadFile(CAFile)
if err != nil { if err != nil {
return nil, err return nil, err
} }
caCertPool := x509.NewCertPool() caCertPool := x509.NewCertPool()
if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { if ok := caCertPool.AppendCertsFromPEM(caCert); !ok {
return nil, errors.New("Failed to add CA to pool") return nil, errors.New("failed to add CA to pool")
} }
tlsConfig := &tls.Config{ tlsConfig := &tls.Config{
RootCAs: caCertPool, RootCAs: caCertPool,
} }
tlsConfig.BuildNameToCertificate() // tlsConfig.BuildNameToCertificate()
return tlsConfig, nil return tlsConfig, nil
} }
@ -104,7 +105,7 @@ func GetJSON(url string, obj interface{}, client *http.Client) (*http.Response,
return resp, errors.New("HTTP status code is not 200") return resp, errors.New("HTTP status code is not 200")
} }
defer resp.Body.Close() defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return resp, err return resp, err
} }
@ -114,10 +115,10 @@ func GetJSON(url string, obj interface{}, client *http.Client) (*http.Response,
// FindAllSubmatchInFile calls re.FindAllSubmatch to find matches in given file // FindAllSubmatchInFile calls re.FindAllSubmatch to find matches in given file
func FindAllSubmatchInFile(fileName string, re *regexp.Regexp) (matches [][][]byte, err error) { func FindAllSubmatchInFile(fileName string, re *regexp.Regexp) (matches [][][]byte, err error) {
if fileName == "/dev/null" { if fileName == "/dev/null" {
err = errors.New("Invalid log file") err = errors.New("invalid log file")
return return
} }
if content, err := ioutil.ReadFile(fileName); err == nil { if content, err := os.ReadFile(fileName); err == nil {
matches = re.FindAllSubmatch(content, -1) matches = re.FindAllSubmatch(content, -1)
// fmt.Printf("FindAllSubmatchInFile: %q\n", matches) // fmt.Printf("FindAllSubmatchInFile: %q\n", matches)
} }
@ -127,7 +128,7 @@ func FindAllSubmatchInFile(fileName string, re *regexp.Regexp) (matches [][][]by
// ExtractSizeFromLog uses a regexp to extract the size from log files // ExtractSizeFromLog uses a regexp to extract the size from log files
func ExtractSizeFromLog(logFile string, re *regexp.Regexp) string { func ExtractSizeFromLog(logFile string, re *regexp.Regexp) string {
matches, _ := FindAllSubmatchInFile(logFile, re) matches, _ := FindAllSubmatchInFile(logFile, re)
if matches == nil || len(matches) == 0 { if len(matches) == 0 {
return "" return ""
} }
// return the first capture group of the last occurrence // return the first capture group of the last occurrence

5
internal/util_test.go
View File

@ -1,7 +1,6 @@
package internal package internal
import ( import (
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@ -29,11 +28,11 @@ sent 7.55M bytes received 823.25M bytes 5.11M bytes/sec
total size is 1.33T speedup is 1,604.11 total size is 1.33T speedup is 1,604.11
` `
Convey("Log parser should work", t, func() { Convey("Log parser should work", t, func() {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
So(err, ShouldBeNil) So(err, ShouldBeNil)
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
logFile := filepath.Join(tmpDir, "rs.log") logFile := filepath.Join(tmpDir, "rs.log")
err = ioutil.WriteFile(logFile, []byte(realLogContent), 0755) err = os.WriteFile(logFile, []byte(realLogContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
res := ExtractSizeFromRsyncLog(logFile) res := ExtractSizeFromRsyncLog(logFile)

5
manager/config_test.go
View File

@ -2,7 +2,6 @@ package manager
import ( import (
"fmt" "fmt"
"io/ioutil"
"os" "os"
"strings" "strings"
"testing" "testing"
@ -37,11 +36,11 @@ func TestConfig(t *testing.T) {
Convey("load Config should work", t, func() { Convey("load Config should work", t, func() {
Convey("create config file & cli context", func() { Convey("create config file & cli context", func() {
tmpfile, err := ioutil.TempFile("", "tunasync") tmpfile, err := os.CreateTemp("", "tunasync")
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
defer os.Remove(tmpfile.Name()) defer os.Remove(tmpfile.Name())
err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644) err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644)
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
defer tmpfile.Close() defer tmpfile.Close()

2
manager/db.go
View File

@ -141,7 +141,7 @@ func (b *kvDBAdapter) ListWorkers() (ws []WorkerStatus, err error) {
func (b *kvDBAdapter) GetWorker(workerID string) (w WorkerStatus, err error) { func (b *kvDBAdapter) GetWorker(workerID string) (w WorkerStatus, err error) {
var v []byte var v []byte
v, err = b.db.Get(_workerBucketKey, workerID) v, _ = b.db.Get(_workerBucketKey, workerID)
if v == nil { if v == nil {
err = fmt.Errorf("invalid workerID %s", workerID) err = fmt.Errorf("invalid workerID %s", workerID)
} else { } else {

16
manager/db_test.go
View File

@ -3,7 +3,6 @@ package manager
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
@ -32,7 +31,7 @@ func DBAdapterTest(db dbAdapter) {
LastOnline: time.Now(), LastOnline: time.Now(),
LastRegister: time.Now(), LastRegister: time.Now(),
} }
w, err = db.CreateWorker(w) _, err = db.CreateWorker(w)
So(err, ShouldBeNil) So(err, ShouldBeNil)
} }
@ -73,7 +72,7 @@ func DBAdapterTest(db dbAdapter) {
Convey("update mirror status", func() { Convey("update mirror status", func() {
status := []MirrorStatus{ status := []MirrorStatus{
MirrorStatus{ {
Name: "arch-sync1", Name: "arch-sync1",
Worker: testWorkerIDs[0], Worker: testWorkerIDs[0],
IsMaster: true, IsMaster: true,
@ -84,7 +83,7 @@ func DBAdapterTest(db dbAdapter) {
Upstream: "mirrors.tuna.tsinghua.edu.cn", Upstream: "mirrors.tuna.tsinghua.edu.cn",
Size: "3GB", Size: "3GB",
}, },
MirrorStatus{ {
Name: "arch-sync2", Name: "arch-sync2",
Worker: testWorkerIDs[1], Worker: testWorkerIDs[1],
IsMaster: true, IsMaster: true,
@ -95,7 +94,7 @@ func DBAdapterTest(db dbAdapter) {
Upstream: "mirrors.tuna.tsinghua.edu.cn", Upstream: "mirrors.tuna.tsinghua.edu.cn",
Size: "4GB", Size: "4GB",
}, },
MirrorStatus{ {
Name: "arch-sync3", Name: "arch-sync3",
Worker: testWorkerIDs[1], Worker: testWorkerIDs[1],
IsMaster: true, IsMaster: true,
@ -159,12 +158,11 @@ func DBAdapterTest(db dbAdapter) {
}) })
}) })
return
} }
func TestDBAdapter(t *testing.T) { func TestDBAdapter(t *testing.T) {
Convey("boltAdapter should work", t, func() { Convey("boltAdapter should work", t, func() {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
@ -200,7 +198,7 @@ func TestDBAdapter(t *testing.T) {
}) })
Convey("badgerAdapter should work", t, func() { Convey("badgerAdapter should work", t, func() {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
@ -218,7 +216,7 @@ func TestDBAdapter(t *testing.T) {
}) })
Convey("leveldbAdapter should work", t, func() { Convey("leveldbAdapter should work", t, func() {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)

4
manager/server.go
View File

@ -267,7 +267,7 @@ func (s *Manager) updateSchedulesOfWorker(c *gin.Context) {
if len(mirrorName) == 0 { if len(mirrorName) == 0 {
s.returnErrJSON( s.returnErrJSON(
c, http.StatusBadRequest, c, http.StatusBadRequest,
errors.New("Mirror Name should not be empty"), errors.New("mirror Name should not be empty"),
) )
} }
@ -312,7 +312,7 @@ func (s *Manager) updateJobOfWorker(c *gin.Context) {
if len(mirrorName) == 0 { if len(mirrorName) == 0 {
s.returnErrJSON( s.returnErrJSON(
c, http.StatusBadRequest, c, http.StatusBadRequest,
errors.New("Mirror Name should not be empty"), errors.New("mirror Name should not be empty"),
) )
} }

14
manager/server_test.go
View File

@ -3,7 +3,7 @@ package manager
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil" "io"
"math/rand" "math/rand"
"net/http" "net/http"
"strings" "strings"
@ -36,7 +36,7 @@ func TestHTTPServer(t *testing.T) {
So(s, ShouldNotBeNil) So(s, ShouldNotBeNil)
s.setDBAdapter(&mockDBAdapter{ s.setDBAdapter(&mockDBAdapter{
workerStore: map[string]WorkerStatus{ workerStore: map[string]WorkerStatus{
_magicBadWorkerID: WorkerStatus{ _magicBadWorkerID: {
ID: _magicBadWorkerID, ID: _magicBadWorkerID,
}}, }},
statusStore: make(map[string]MirrorStatus), statusStore: make(map[string]MirrorStatus),
@ -48,7 +48,7 @@ func TestHTTPServer(t *testing.T) {
So(resp.StatusCode, ShouldEqual, http.StatusOK) So(resp.StatusCode, ShouldEqual, http.StatusOK)
So(resp.Header.Get("Content-Type"), ShouldEqual, "application/json; charset=utf-8") So(resp.Header.Get("Content-Type"), ShouldEqual, "application/json; charset=utf-8")
defer resp.Body.Close() defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body) body, err := io.ReadAll(resp.Body)
So(err, ShouldBeNil) So(err, ShouldBeNil)
var p map[string]string var p map[string]string
err = json.Unmarshal(body, &p) err = json.Unmarshal(body, &p)
@ -268,8 +268,8 @@ func TestHTTPServer(t *testing.T) {
Convey("Update schedule of valid mirrors", func(ctx C) { Convey("Update schedule of valid mirrors", func(ctx C) {
msg := MirrorSchedules{ msg := MirrorSchedules{
Schedules: []MirrorSchedule{ Schedules: []MirrorSchedule{
MirrorSchedule{MirrorName: "arch-sync1", NextSchedule: time.Now().Add(time.Minute * 10)}, {MirrorName: "arch-sync1", NextSchedule: time.Now().Add(time.Minute * 10)},
MirrorSchedule{MirrorName: "arch-sync2", NextSchedule: time.Now().Add(time.Minute * 7)}, {MirrorName: "arch-sync2", NextSchedule: time.Now().Add(time.Minute * 7)},
}, },
} }
@ -346,8 +346,8 @@ func TestHTTPServer(t *testing.T) {
invalidWorker := "test_worker2" invalidWorker := "test_worker2"
sch := MirrorSchedules{ sch := MirrorSchedules{
Schedules: []MirrorSchedule{ Schedules: []MirrorSchedule{
MirrorSchedule{MirrorName: "arch-sync1", NextSchedule: time.Now().Add(time.Minute * 10)}, {MirrorName: "arch-sync1", NextSchedule: time.Now().Add(time.Minute * 10)},
MirrorSchedule{MirrorName: "arch-sync2", NextSchedule: time.Now().Add(time.Minute * 7)}, {MirrorName: "arch-sync2", NextSchedule: time.Now().Add(time.Minute * 7)},
}, },
} }
resp, err := PostJSON(fmt.Sprintf("%s/workers/%s/schedules", resp, err := PostJSON(fmt.Sprintf("%s/workers/%s/schedules",

76
worker/cgroup.go
View File

@ -3,7 +3,7 @@ package worker
import ( import (
"errors" "errors"
"fmt" "fmt"
"io/ioutil" "io"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
@ -12,32 +12,32 @@ import (
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
"github.com/moby/moby/pkg/reexec"
cgv1 "github.com/containerd/cgroups" cgv1 "github.com/containerd/cgroups"
cgv2 "github.com/containerd/cgroups/v2" cgv2 "github.com/containerd/cgroups/v2"
"github.com/moby/moby/pkg/reexec"
contspecs "github.com/opencontainers/runtime-spec/specs-go" contspecs "github.com/opencontainers/runtime-spec/specs-go"
) )
type cgroupHook struct { type cgroupHook struct {
emptyHook emptyHook
cgCfg cgroupConfig cgCfg cgroupConfig
memLimit MemBytes memLimit MemBytes
cgMgrV1 cgv1.Cgroup cgMgrV1 cgv1.Cgroup
cgMgrV2 *cgv2.Manager cgMgrV2 *cgv2.Manager
} }
type execCmd string type execCmd string
const ( const (
cmdCont execCmd = "cont" cmdCont execCmd = "cont"
cmdAbrt execCmd = "abrt" cmdAbrt execCmd = "abrt"
) )
func init () { func init() {
reexec.Register("tunasync-exec", waitExec) reexec.Register("tunasync-exec", waitExec)
} }
func waitExec () { func waitExec() {
binary, err := exec.LookPath(os.Args[1]) binary, err := exec.LookPath(os.Args[1])
if err != nil { if err != nil {
panic(err) panic(err)
@ -46,7 +46,7 @@ func waitExec () {
pipe := os.NewFile(3, "pipe") pipe := os.NewFile(3, "pipe")
if pipe != nil { if pipe != nil {
if _, err := pipe.Stat(); err == nil { if _, err := pipe.Stat(); err == nil {
cmdBytes, err := ioutil.ReadAll(pipe) cmdBytes, err := io.ReadAll(pipe)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -54,11 +54,11 @@ func waitExec () {
} }
cmd := execCmd(string(cmdBytes)) cmd := execCmd(string(cmdBytes))
switch cmd { switch cmd {
case cmdAbrt: case cmdAbrt:
fallthrough fallthrough
default: default:
panic("Exited on request") panic("Exited on request")
case cmdCont: case cmdCont:
} }
} }
} }
@ -71,7 +71,7 @@ func waitExec () {
panic("Exec failed.") panic("Exec failed.")
} }
func initCgroup(cfg *cgroupConfig) (error) { func initCgroup(cfg *cgroupConfig) error {
logger.Debugf("Initializing cgroup") logger.Debugf("Initializing cgroup")
baseGroup := cfg.Group baseGroup := cfg.Group
@ -103,7 +103,7 @@ func initCgroup(cfg *cgroupConfig) (error) {
} }
if baseGroup == "" { if baseGroup == "" {
logger.Debugf("Creating a sub group and move all processes into it") logger.Debugf("Creating a sub group and move all processes into it")
wkrMgr, err := cfg.cgMgrV2.NewChild("__worker", nil); wkrMgr, err := cfg.cgMgrV2.NewChild("__worker", nil)
if err != nil { if err != nil {
return err return err
} }
@ -117,8 +117,8 @@ func initCgroup(cfg *cgroupConfig) (error) {
if len(procs) == 0 { if len(procs) == 0 {
break break
} }
for _, p := range(procs) { for _, p := range procs {
if err := wkrMgr.AddProc(p); err != nil{ if err := wkrMgr.AddProc(p); err != nil {
if errors.Is(err, syscall.ESRCH) { if errors.Is(err, syscall.ESRCH) {
logger.Debugf("Write pid %d to sub group failed: process vanished, ignoring") logger.Debugf("Write pid %d to sub group failed: process vanished, ignoring")
} else { } else {
@ -129,7 +129,7 @@ func initCgroup(cfg *cgroupConfig) (error) {
} }
} else { } else {
logger.Debugf("Trying to create a sub group in that group") logger.Debugf("Trying to create a sub group in that group")
testMgr, err := cfg.cgMgrV2.NewChild("__test", nil); testMgr, err := cfg.cgMgrV2.NewChild("__test", nil)
if err != nil { if err != nil {
logger.Errorf("Cannot create a sub group in the cgroup") logger.Errorf("Cannot create a sub group in the cgroup")
return err return err
@ -152,9 +152,9 @@ func initCgroup(cfg *cgroupConfig) (error) {
if baseGroup != "" { if baseGroup != "" {
pather = cgv1.StaticPath(baseGroup) pather = cgv1.StaticPath(baseGroup)
} else { } else {
pather = (func(p cgv1.Path) (cgv1.Path){ pather = (func(p cgv1.Path) cgv1.Path {
return func(subsys cgv1.Name) (string, error){ return func(subsys cgv1.Name) (string, error) {
path, err := p(subsys); path, err := p(subsys)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -167,14 +167,14 @@ func initCgroup(cfg *cgroupConfig) (error) {
} }
logger.Infof("Loading cgroup") logger.Infof("Loading cgroup")
var err error var err error
if cfg.cgMgrV1, err = cgv1.Load(cgv1.V1, pather, func(cfg *cgv1.InitConfig) error{ if cfg.cgMgrV1, err = cgv1.Load(cgv1.V1, pather, func(cfg *cgv1.InitConfig) error {
cfg.InitCheck = cgv1.AllowAny cfg.InitCheck = cgv1.AllowAny
return nil return nil
}); err != nil { }); err != nil {
return err return err
} }
logger.Debugf("Available subsystems:") logger.Debugf("Available subsystems:")
for _, subsys := range(cfg.cgMgrV1.Subsystems()) { for _, subsys := range cfg.cgMgrV1.Subsystems() {
p, err := pather(subsys.Name()) p, err := pather(subsys.Name())
if err != nil { if err != nil {
return err return err
@ -183,11 +183,11 @@ func initCgroup(cfg *cgroupConfig) (error) {
} }
if baseGroup == "" { if baseGroup == "" {
logger.Debugf("Creating a sub group and move all processes into it") logger.Debugf("Creating a sub group and move all processes into it")
wkrMgr, err := cfg.cgMgrV1.New("__worker", &contspecs.LinuxResources{}); wkrMgr, err := cfg.cgMgrV1.New("__worker", &contspecs.LinuxResources{})
if err != nil { if err != nil {
return err return err
} }
for _, subsys := range(cfg.cgMgrV1.Subsystems()) { for _, subsys := range cfg.cgMgrV1.Subsystems() {
logger.Debugf("Reading pids for subsystem %s", subsys.Name()) logger.Debugf("Reading pids for subsystem %s", subsys.Name())
for { for {
procs, err := cfg.cgMgrV1.Processes(subsys.Name(), false) procs, err := cfg.cgMgrV1.Processes(subsys.Name(), false)
@ -202,7 +202,7 @@ func initCgroup(cfg *cgroupConfig) (error) {
if len(procs) == 0 { if len(procs) == 0 {
break break
} }
for _, proc := range(procs) { for _, proc := range procs {
if err := wkrMgr.Add(proc); err != nil { if err := wkrMgr.Add(proc); err != nil {
if errors.Is(err, syscall.ESRCH) { if errors.Is(err, syscall.ESRCH) {
logger.Debugf("Write pid %d to sub group failed: process vanished, ignoring") logger.Debugf("Write pid %d to sub group failed: process vanished, ignoring")
@ -215,7 +215,7 @@ func initCgroup(cfg *cgroupConfig) (error) {
} }
} else { } else {
logger.Debugf("Trying to create a sub group in that group") logger.Debugf("Trying to create a sub group in that group")
testMgr, err := cfg.cgMgrV1.New("__test", &contspecs.LinuxResources{}); testMgr, err := cfg.cgMgrV1.New("__test", &contspecs.LinuxResources{})
if err != nil { if err != nil {
logger.Errorf("Cannot create a sub group in the cgroup") logger.Errorf("Cannot create a sub group in the cgroup")
return err return err
@ -223,7 +223,7 @@ func initCgroup(cfg *cgroupConfig) (error) {
if err := testMgr.Delete(); err != nil { if err := testMgr.Delete(); err != nil {
return err return err
} }
for _, subsys := range(cfg.cgMgrV1.Subsystems()) { for _, subsys := range cfg.cgMgrV1.Subsystems() {
logger.Debugf("Reading pids for subsystem %s", subsys.Name()) logger.Debugf("Reading pids for subsystem %s", subsys.Name())
procs, err := cfg.cgMgrV1.Processes(subsys.Name(), false) procs, err := cfg.cgMgrV1.Processes(subsys.Name(), false)
if err != nil { if err != nil {
@ -253,7 +253,7 @@ func newCgroupHook(p mirrorProvider, cfg cgroupConfig, memLimit MemBytes) *cgrou
emptyHook: emptyHook{ emptyHook: emptyHook{
provider: p, provider: p,
}, },
cgCfg: cfg, cgCfg: cfg,
memLimit: memLimit, memLimit: memLimit,
} }
} }
@ -263,7 +263,7 @@ func (c *cgroupHook) preExec() error {
logger.Debugf("Creating v2 cgroup for task %s", c.provider.Name()) logger.Debugf("Creating v2 cgroup for task %s", c.provider.Name())
var resSet *cgv2.Resources var resSet *cgv2.Resources
if c.memLimit != 0 { if c.memLimit != 0 {
resSet = &cgv2.Resources { resSet = &cgv2.Resources{
Memory: &cgv2.Memory{ Memory: &cgv2.Memory{
Max: func(i int64) *int64 { return &i }(c.memLimit.Value()), Max: func(i int64) *int64 { return &i }(c.memLimit.Value()),
}, },
@ -279,7 +279,7 @@ func (c *cgroupHook) preExec() error {
logger.Debugf("Creating v1 cgroup for task %s", c.provider.Name()) logger.Debugf("Creating v1 cgroup for task %s", c.provider.Name())
var resSet contspecs.LinuxResources var resSet contspecs.LinuxResources
if c.memLimit != 0 { if c.memLimit != 0 {
resSet = contspecs.LinuxResources { resSet = contspecs.LinuxResources{
Memory: &contspecs.LinuxMemory{ Memory: &contspecs.LinuxMemory{
Limit: func(i int64) *int64 { return &i }(c.memLimit.Value()), Limit: func(i int64) *int64 { return &i }(c.memLimit.Value()),
}, },
@ -334,7 +334,7 @@ func (c *cgroupHook) killAll() error {
taskList := []int{} taskList := []int{}
if c.cgCfg.isUnified { if c.cgCfg.isUnified {
procs, err := c.cgMgrV2.Procs(false) procs, err := c.cgMgrV2.Procs(false)
if (err != nil) { if err != nil {
return []int{}, err return []int{}, err
} }
for _, proc := range procs { for _, proc := range procs {
@ -342,16 +342,16 @@ func (c *cgroupHook) killAll() error {
} }
} else { } else {
taskSet := make(map[int]struct{}) taskSet := make(map[int]struct{})
for _, subsys := range(c.cgMgrV1.Subsystems()) { for _, subsys := range c.cgMgrV1.Subsystems() {
procs, err := c.cgMgrV1.Processes(subsys.Name(), false) procs, err := c.cgMgrV1.Processes(subsys.Name(), false)
if err != nil { if err != nil {
return []int{}, err return []int{}, err
} }
for _, proc := range(procs) { for _, proc := range procs {
taskSet[proc.Pid] = struct{}{} taskSet[proc.Pid] = struct{}{}
} }
} }
for proc := range(taskSet) { for proc := range taskSet {
taskList = append(taskList, proc) taskList = append(taskList, proc)
} }
} }

436
worker/cgroup_test.go
View File

@ -1,16 +1,16 @@
package worker package worker
import ( import (
"io/ioutil" "errors"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
"syscall"
"testing" "testing"
"time" "time"
"errors"
"syscall"
cgv1 "github.com/containerd/cgroups" cgv1 "github.com/containerd/cgroups"
cgv2 "github.com/containerd/cgroups/v2" cgv2 "github.com/containerd/cgroups/v2"
units "github.com/docker/go-units" units "github.com/docker/go-units"
@ -21,14 +21,14 @@ import (
func init() { func init() {
_, testReexec := os.LookupEnv("TESTREEXEC") _, testReexec := os.LookupEnv("TESTREEXEC")
if ! testReexec { if !testReexec {
reexec.Init() reexec.Init()
} }
} }
func TestReexec(t *testing.T) { func TestReexec(t *testing.T) {
testCase, testReexec := os.LookupEnv("TESTREEXEC") testCase, testReexec := os.LookupEnv("TESTREEXEC")
if ! testReexec { if !testReexec {
return return
} }
for len(os.Args) > 1 { for len(os.Args) > 1 {
@ -39,51 +39,51 @@ func TestReexec(t *testing.T) {
} }
} }
switch testCase { switch testCase {
case "1": case "1":
Convey("Reexec should panic when command not found", t, func(ctx C){ Convey("Reexec should panic when command not found", t, func(ctx C) {
So(func(){ So(func() {
reexec.Init() reexec.Init()
}, ShouldPanicWith, exec.ErrNotFound) }, ShouldPanicWith, exec.ErrNotFound)
}) })
case "2": case "2":
Convey("Reexec should run when fd 3 is not open", t, func(ctx C){ Convey("Reexec should run when fd 3 is not open", t, func(ctx C) {
So((func() error{ So((func() error {
pipe := os.NewFile(3, "pipe") pipe := os.NewFile(3, "pipe")
if pipe == nil { if pipe == nil {
return errors.New("pipe is nil") return errors.New("pipe is nil")
} else { } else {
_, err := pipe.Stat() _, err := pipe.Stat()
return err return err
} }
})(), ShouldNotBeNil) })(), ShouldNotBeNil)
So(func(){ So(func() {
reexec.Init() reexec.Init()
}, ShouldPanicWith, syscall.ENOEXEC) }, ShouldPanicWith, syscall.ENOEXEC)
}) })
case "3": case "3":
Convey("Reexec should fail when fd 3 is sent with abrt cmd", t, func(ctx C){ Convey("Reexec should fail when fd 3 is sent with abrt cmd", t, func(ctx C) {
So(func(){ So(func() {
reexec.Init() reexec.Init()
}, ShouldPanicWith, "Exited on request") }, ShouldPanicWith, "Exited on request")
}) })
case "4": case "4":
Convey("Reexec should run when fd 3 is sent with cont cmd", t, func(ctx C){ Convey("Reexec should run when fd 3 is sent with cont cmd", t, func(ctx C) {
So(func(){ So(func() {
reexec.Init() reexec.Init()
}, ShouldPanicWith, syscall.ENOEXEC) }, ShouldPanicWith, syscall.ENOEXEC)
}) })
case "5": case "5":
Convey("Reexec should not be triggered when argv[0] is not reexec", t, func(ctx C){ Convey("Reexec should not be triggered when argv[0] is not reexec", t, func(ctx C) {
So(func(){ So(func() {
reexec.Init() reexec.Init()
}, ShouldNotPanic) }, ShouldNotPanic)
}) })
} }
} }
func TestCgroup(t *testing.T) { func TestCgroup(t *testing.T) {
var cgcf *cgroupConfig var cgcf *cgroupConfig
Convey("init cgroup", t, func(ctx C){ Convey("init cgroup", t, func(ctx C) {
_, useCurrentCgroup := os.LookupEnv("USECURCGROUP") _, useCurrentCgroup := os.LookupEnv("USECURCGROUP")
cgcf = &cgroupConfig{BasePath: "/sys/fs/cgroup", Group: "tunasync", Subsystem: "cpu"} cgcf = &cgroupConfig{BasePath: "/sys/fs/cgroup", Group: "tunasync", Subsystem: "cpu"}
if useCurrentCgroup { if useCurrentCgroup {
@ -97,28 +97,28 @@ func TestCgroup(t *testing.T) {
So(cgcf.cgMgrV1, ShouldNotBeNil) So(cgcf.cgMgrV1, ShouldNotBeNil)
} }
Convey("Cgroup Should Work", func(ctx C) { Convey("Cgroup Should Work", func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
cmdScript := filepath.Join(tmpDir, "cmd.sh") cmdScript := filepath.Join(tmpDir, "cmd.sh")
daemonScript := filepath.Join(tmpDir, "daemon.sh") daemonScript := filepath.Join(tmpDir, "daemon.sh")
tmpFile := filepath.Join(tmpDir, "log_file") tmpFile := filepath.Join(tmpDir, "log_file")
bgPidfile := filepath.Join(tmpDir, "bg.pid") bgPidfile := filepath.Join(tmpDir, "bg.pid")
c := cmdConfig{ c := cmdConfig{
name: "tuna-cgroup", name: "tuna-cgroup",
upstreamURL: "http://mirrors.tuna.moe/", upstreamURL: "http://mirrors.tuna.moe/",
command: cmdScript + " " + daemonScript, command: cmdScript + " " + daemonScript,
workingDir: tmpDir, workingDir: tmpDir,
logDir: tmpDir, logDir: tmpDir,
logFile: tmpFile, logFile: tmpFile,
interval: 600 * time.Second, interval: 600 * time.Second,
env: map[string]string{ env: map[string]string{
"BG_PIDFILE": bgPidfile, "BG_PIDFILE": bgPidfile,
}, },
} }
cmdScriptContent := `#!/bin/bash cmdScriptContent := `#!/bin/bash
redirect-std() { redirect-std() {
[[ -t 0 ]] && exec </dev/null [[ -t 0 ]] && exec </dev/null
[[ -t 1 ]] && exec >/dev/null [[ -t 1 ]] && exec >/dev/null
@ -144,167 +144,127 @@ echo $$
daemonize $@ daemonize $@
sleep 5 sleep 5
` `
daemonScriptContent := `#!/bin/bash daemonScriptContent := `#!/bin/bash
echo $$ > $BG_PIDFILE echo $$ > $BG_PIDFILE
sleep 30 sleep 30
` `
err = ioutil.WriteFile(cmdScript, []byte(cmdScriptContent), 0755) err = os.WriteFile(cmdScript, []byte(cmdScriptContent), 0755)
So(err, ShouldBeNil)
err = ioutil.WriteFile(daemonScript, []byte(daemonScriptContent), 0755)
So(err, ShouldBeNil)
provider, err := newCmdProvider(c)
So(err, ShouldBeNil)
cg := newCgroupHook(provider, *cgcf, 0)
provider.AddHook(cg)
err = cg.preExec()
So(err, ShouldBeNil)
go func() {
err := provider.Run(make(chan empty, 1))
ctx.So(err, ShouldNotBeNil)
}()
time.Sleep(1 * time.Second)
// Deamon should be started
daemonPidBytes, err := ioutil.ReadFile(bgPidfile)
So(err, ShouldBeNil)
daemonPid := strings.Trim(string(daemonPidBytes), " \n")
logger.Debug("daemon pid: %s", daemonPid)
procDir := filepath.Join("/proc", daemonPid)
_, err = os.Stat(procDir)
So(err, ShouldBeNil)
err = provider.Terminate()
So(err, ShouldBeNil)
// Deamon won't be killed
_, err = os.Stat(procDir)
So(err, ShouldBeNil)
// Deamon can be killed by cgroup killer
cg.postExec()
_, err = os.Stat(procDir)
So(os.IsNotExist(err), ShouldBeTrue)
})
Convey("Rsync Memory Should Be Limited", func() {
tmpDir, err := ioutil.TempDir("", "tunasync")
defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil)
scriptFile := filepath.Join(tmpDir, "myrsync")
tmpFile := filepath.Join(tmpDir, "log_file")
c := rsyncConfig{
name: "tuna-cgroup",
upstreamURL: "rsync://rsync.tuna.moe/tuna/",
rsyncCmd: scriptFile,
workingDir: tmpDir,
logDir: tmpDir,
logFile: tmpFile,
useIPv6: true,
interval: 600 * time.Second,
}
provider, err := newRsyncProvider(c)
So(err, ShouldBeNil)
cg := newCgroupHook(provider, *cgcf, 512 * units.MiB)
provider.AddHook(cg)
err = cg.preExec()
So(err, ShouldBeNil)
if cgcf.isUnified {
cgpath := filepath.Join(cgcf.BasePath, cgcf.Group, provider.Name())
if useCurrentCgroup {
group, err := cgv2.NestedGroupPath(filepath.Join("..", provider.Name()))
So(err, ShouldBeNil)
cgpath = filepath.Join(cgcf.BasePath, group)
}
memoLimit, err := ioutil.ReadFile(filepath.Join(cgpath, "memory.max"))
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(strings.Trim(string(memoLimit), "\n"), ShouldEqual, strconv.Itoa(512*1024*1024)) err = os.WriteFile(daemonScript, []byte(daemonScriptContent), 0755)
} else { So(err, ShouldBeNil)
for _, subsys := range(cg.cgMgrV1.Subsystems()) {
if subsys.Name() == cgv1.Memory { provider, err := newCmdProvider(c)
cgpath := filepath.Join(cgcf.Group, provider.Name()) So(err, ShouldBeNil)
if useCurrentCgroup {
p, err := cgv1.NestedPath(filepath.Join("..", provider.Name()))(cgv1.Memory) cg := newCgroupHook(provider, *cgcf, 0)
provider.AddHook(cg)
err = cg.preExec()
So(err, ShouldBeNil)
go func() {
err := provider.Run(make(chan empty, 1))
ctx.So(err, ShouldNotBeNil)
}()
time.Sleep(1 * time.Second)
// Deamon should be started
daemonPidBytes, err := os.ReadFile(bgPidfile)
So(err, ShouldBeNil)
daemonPid := strings.Trim(string(daemonPidBytes), " \n")
logger.Debug("daemon pid: %s", daemonPid)
procDir := filepath.Join("/proc", daemonPid)
_, err = os.Stat(procDir)
So(err, ShouldBeNil)
err = provider.Terminate()
So(err, ShouldBeNil)
// Deamon won't be killed
_, err = os.Stat(procDir)
So(err, ShouldBeNil)
// Deamon can be killed by cgroup killer
cg.postExec()
_, err = os.Stat(procDir)
So(os.IsNotExist(err), ShouldBeTrue)
})
Convey("Rsync Memory Should Be Limited", func() {
tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil)
scriptFile := filepath.Join(tmpDir, "myrsync")
tmpFile := filepath.Join(tmpDir, "log_file")
c := rsyncConfig{
name: "tuna-cgroup",
upstreamURL: "rsync://rsync.tuna.moe/tuna/",
rsyncCmd: scriptFile,
workingDir: tmpDir,
logDir: tmpDir,
logFile: tmpFile,
useIPv6: true,
interval: 600 * time.Second,
}
provider, err := newRsyncProvider(c)
So(err, ShouldBeNil)
cg := newCgroupHook(provider, *cgcf, 512*units.MiB)
provider.AddHook(cg)
err = cg.preExec()
So(err, ShouldBeNil)
if cgcf.isUnified {
cgpath := filepath.Join(cgcf.BasePath, cgcf.Group, provider.Name())
if useCurrentCgroup {
group, err := cgv2.NestedGroupPath(filepath.Join("..", provider.Name()))
So(err, ShouldBeNil)
cgpath = filepath.Join(cgcf.BasePath, group)
}
memoLimit, err := os.ReadFile(filepath.Join(cgpath, "memory.max"))
So(err, ShouldBeNil)
So(strings.Trim(string(memoLimit), "\n"), ShouldEqual, strconv.Itoa(512*1024*1024))
} else {
for _, subsys := range cg.cgMgrV1.Subsystems() {
if subsys.Name() == cgv1.Memory {
cgpath := filepath.Join(cgcf.Group, provider.Name())
if useCurrentCgroup {
p, err := cgv1.NestedPath(filepath.Join("..", provider.Name()))(cgv1.Memory)
So(err, ShouldBeNil)
cgpath = p
}
memoLimit, err := os.ReadFile(filepath.Join(cgcf.BasePath, "memory", cgpath, "memory.limit_in_bytes"))
So(err, ShouldBeNil) So(err, ShouldBeNil)
cgpath = p So(strings.Trim(string(memoLimit), "\n"), ShouldEqual, strconv.Itoa(512*1024*1024))
}
memoLimit, err := ioutil.ReadFile(filepath.Join(cgcf.BasePath, "memory", cgpath, "memory.limit_in_bytes"))
So(err, ShouldBeNil)
So(strings.Trim(string(memoLimit), "\n"), ShouldEqual, strconv.Itoa(512*1024*1024))
}
}
}
cg.postExec()
So(cg.cgMgrV1, ShouldBeNil)
})
Reset(func() {
if cgcf.isUnified {
if cgcf.Group == "" {
wkrg, err := cgv2.NestedGroupPath("");
So(err, ShouldBeNil)
wkrMgr, err := cgv2.LoadManager("/sys/fs/cgroup", wkrg);
allCtrls, err := wkrMgr.Controllers()
So(err, ShouldBeNil)
err = wkrMgr.ToggleControllers(allCtrls, cgv2.Disable)
So(err, ShouldBeNil)
origMgr := cgcf.cgMgrV2
for {
logger.Debugf("Restoring pids")
procs, err := wkrMgr.Procs(false)
So(err, ShouldBeNil)
if len(procs) == 0 {
break
}
for _, p := range(procs) {
if err := origMgr.AddProc(p); err != nil{
if errors.Is(err, syscall.ESRCH) {
logger.Debugf("Write pid %d to sub group failed: process vanished, ignoring")
} else {
So(err, ShouldBeNil)
}
}
} }
} }
err = wkrMgr.Delete()
So(err, ShouldBeNil)
} }
} else { cg.postExec()
if cgcf.Group == "" { So(cg.cgMgrV1, ShouldBeNil)
pather := (func(p cgv1.Path) (cgv1.Path){ })
return func(subsys cgv1.Name) (string, error){ Reset(func() {
path, err := p(subsys); if cgcf.isUnified {
if err != nil { if cgcf.Group == "" {
return "", err wkrg, err := cgv2.NestedGroupPath("")
} So(err, ShouldBeNil)
if path == "/" { wkrMgr, err := cgv2.LoadManager("/sys/fs/cgroup", wkrg)
return "", cgv1.ErrControllerNotActive allCtrls, err := wkrMgr.Controllers()
} So(err, ShouldBeNil)
return path, err err = wkrMgr.ToggleControllers(allCtrls, cgv2.Disable)
} So(err, ShouldBeNil)
})(cgv1.NestedPath("")) origMgr := cgcf.cgMgrV2
wkrMgr, err := cgv1.Load(cgv1.V1, pather, func(cfg *cgv1.InitConfig) error{
cfg.InitCheck = cgv1.AllowAny
return nil
})
So(err, ShouldBeNil)
origMgr := cgcf.cgMgrV1
for _, subsys := range(wkrMgr.Subsystems()){
for { for {
procs, err := wkrMgr.Processes(subsys.Name(), false) logger.Debugf("Restoring pids")
procs, err := wkrMgr.Procs(false)
So(err, ShouldBeNil) So(err, ShouldBeNil)
if len(procs) == 0 { if len(procs) == 0 {
break break
} }
for _, proc := range(procs) { for _, p := range procs {
if err := origMgr.Add(proc); err != nil { if err := origMgr.AddProc(p); err != nil {
if errors.Is(err, syscall.ESRCH) { if errors.Is(err, syscall.ESRCH) {
logger.Debugf("Write pid %d to sub group failed: process vanished, ignoring") logger.Debugf("Write pid %d to sub group failed: process vanished, ignoring")
} else { } else {
@ -313,11 +273,51 @@ sleep 30
} }
} }
} }
err = wkrMgr.Delete()
So(err, ShouldBeNil)
}
} else {
if cgcf.Group == "" {
pather := (func(p cgv1.Path) cgv1.Path {
return func(subsys cgv1.Name) (string, error) {
path, err := p(subsys)
if err != nil {
return "", err
}
if path == "/" {
return "", cgv1.ErrControllerNotActive
}
return path, err
}
})(cgv1.NestedPath(""))
wkrMgr, err := cgv1.Load(cgv1.V1, pather, func(cfg *cgv1.InitConfig) error {
cfg.InitCheck = cgv1.AllowAny
return nil
})
So(err, ShouldBeNil)
origMgr := cgcf.cgMgrV1
for _, subsys := range wkrMgr.Subsystems() {
for {
procs, err := wkrMgr.Processes(subsys.Name(), false)
So(err, ShouldBeNil)
if len(procs) == 0 {
break
}
for _, proc := range procs {
if err := origMgr.Add(proc); err != nil {
if errors.Is(err, syscall.ESRCH) {
logger.Debugf("Write pid %d to sub group failed: process vanished, ignoring")
} else {
So(err, ShouldBeNil)
}
}
}
}
}
err = wkrMgr.Delete()
So(err, ShouldBeNil)
} }
err = wkrMgr.Delete()
So(err, ShouldBeNil)
} }
} })
})
}) })
} }

34
worker/config_test.go
View File

@ -2,11 +2,11 @@ package worker
import ( import (
"fmt" "fmt"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
"time" "time"
units "github.com/docker/go-units" units "github.com/docker/go-units"
. "github.com/smartystreets/goconvey/convey" . "github.com/smartystreets/goconvey/convey"
@ -76,11 +76,11 @@ exec_on_failure = [
}) })
Convey("Everything should work on valid config file", t, func() { Convey("Everything should work on valid config file", t, func() {
tmpfile, err := ioutil.TempFile("", "tunasync") tmpfile, err := os.CreateTemp("", "tunasync")
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
defer os.Remove(tmpfile.Name()) defer os.Remove(tmpfile.Name())
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
So(err, ShouldBeNil) So(err, ShouldBeNil)
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
@ -92,7 +92,7 @@ exec_on_failure = [
curCfgBlob := cfgBlob + incSection curCfgBlob := cfgBlob + incSection
err = ioutil.WriteFile(tmpfile.Name(), []byte(curCfgBlob), 0644) err = os.WriteFile(tmpfile.Name(), []byte(curCfgBlob), 0644)
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
defer tmpfile.Close() defer tmpfile.Close()
@ -116,9 +116,9 @@ provider = "two-stage-rsync"
stage1_profile = "debian" stage1_profile = "debian"
use_ipv6 = true use_ipv6 = true
` `
err = ioutil.WriteFile(filepath.Join(tmpDir, "debian.conf"), []byte(incBlob1), 0644) err = os.WriteFile(filepath.Join(tmpDir, "debian.conf"), []byte(incBlob1), 0644)
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
err = ioutil.WriteFile(filepath.Join(tmpDir, "ubuntu.conf"), []byte(incBlob2), 0644) err = os.WriteFile(filepath.Join(tmpDir, "ubuntu.conf"), []byte(incBlob2), 0644)
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
cfg, err := LoadConfig(tmpfile.Name()) cfg, err := LoadConfig(tmpfile.Name())
@ -145,20 +145,20 @@ use_ipv6 = true
So(m.Name, ShouldEqual, "debian") So(m.Name, ShouldEqual, "debian")
So(m.MirrorDir, ShouldEqual, "") So(m.MirrorDir, ShouldEqual, "")
So(m.Provider, ShouldEqual, provTwoStageRsync) So(m.Provider, ShouldEqual, provTwoStageRsync)
So(m.MemoryLimit.Value(), ShouldEqual, 256 * units.MiB) So(m.MemoryLimit.Value(), ShouldEqual, 256*units.MiB)
m = cfg.Mirrors[2] m = cfg.Mirrors[2]
So(m.Name, ShouldEqual, "fedora") So(m.Name, ShouldEqual, "fedora")
So(m.MirrorDir, ShouldEqual, "") So(m.MirrorDir, ShouldEqual, "")
So(m.Provider, ShouldEqual, provRsync) So(m.Provider, ShouldEqual, provRsync)
So(m.ExcludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt") So(m.ExcludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt")
So(m.MemoryLimit.Value(), ShouldEqual, 128 * units.MiB) So(m.MemoryLimit.Value(), ShouldEqual, 128*units.MiB)
m = cfg.Mirrors[3] m = cfg.Mirrors[3]
So(m.Name, ShouldEqual, "debian-cd") So(m.Name, ShouldEqual, "debian-cd")
So(m.MirrorDir, ShouldEqual, "") So(m.MirrorDir, ShouldEqual, "")
So(m.Provider, ShouldEqual, provTwoStageRsync) So(m.Provider, ShouldEqual, provTwoStageRsync)
So(m.MemoryLimit.Value(), ShouldEqual, 0) So(m.MemoryLimit.Value(), ShouldEqual, 0)
m = cfg.Mirrors[4] m = cfg.Mirrors[4]
So(m.Name, ShouldEqual, "debian-security") So(m.Name, ShouldEqual, "debian-security")
@ -170,11 +170,11 @@ use_ipv6 = true
}) })
Convey("Everything should work on nested config file", t, func() { Convey("Everything should work on nested config file", t, func() {
tmpfile, err := ioutil.TempFile("", "tunasync") tmpfile, err := os.CreateTemp("", "tunasync")
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
defer os.Remove(tmpfile.Name()) defer os.Remove(tmpfile.Name())
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
So(err, ShouldBeNil) So(err, ShouldBeNil)
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
@ -186,7 +186,7 @@ use_ipv6 = true
curCfgBlob := cfgBlob + incSection curCfgBlob := cfgBlob + incSection
err = ioutil.WriteFile(tmpfile.Name(), []byte(curCfgBlob), 0644) err = os.WriteFile(tmpfile.Name(), []byte(curCfgBlob), 0644)
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
defer tmpfile.Close() defer tmpfile.Close()
@ -212,7 +212,7 @@ use_ipv6 = true
provider = "rsync" provider = "rsync"
upstream = "rsync://test.host3/debian-cd/" upstream = "rsync://test.host3/debian-cd/"
` `
err = ioutil.WriteFile(filepath.Join(tmpDir, "nest.conf"), []byte(incBlob1), 0644) err = os.WriteFile(filepath.Join(tmpDir, "nest.conf"), []byte(incBlob1), 0644)
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
cfg, err := LoadConfig(tmpfile.Name()) cfg, err := LoadConfig(tmpfile.Name())
@ -266,11 +266,11 @@ use_ipv6 = true
So(len(cfg.Mirrors), ShouldEqual, 6) So(len(cfg.Mirrors), ShouldEqual, 6)
}) })
Convey("Providers can be inited from a valid config file", t, func() { Convey("Providers can be inited from a valid config file", t, func() {
tmpfile, err := ioutil.TempFile("", "tunasync") tmpfile, err := os.CreateTemp("", "tunasync")
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
defer os.Remove(tmpfile.Name()) defer os.Remove(tmpfile.Name())
err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644) err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644)
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
defer tmpfile.Close() defer tmpfile.Close()
@ -317,7 +317,7 @@ use_ipv6 = true
}) })
Convey("MirrorSubdir should work", t, func() { Convey("MirrorSubdir should work", t, func() {
tmpfile, err := ioutil.TempFile("", "tunasync") tmpfile, err := os.CreateTemp("", "tunasync")
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
defer os.Remove(tmpfile.Name()) defer os.Remove(tmpfile.Name())
@ -363,7 +363,7 @@ use_ipv6 = true
provider = "rsync" provider = "rsync"
upstream = "rsync://test.host3/debian-cd/" upstream = "rsync://test.host3/debian-cd/"
` `
err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644) err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
So(err, ShouldEqual, nil) So(err, ShouldEqual, nil)
defer tmpfile.Close() defer tmpfile.Close()

8
worker/docker_test.go
View File

@ -2,12 +2,12 @@ package worker
import ( import (
"fmt" "fmt"
"io/ioutil"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"testing" "testing"
"time" "time"
units "github.com/docker/go-units" units "github.com/docker/go-units"
"github.com/codeskyblue/go-sh" "github.com/codeskyblue/go-sh"
@ -40,7 +40,7 @@ func getDockerByName(name string) (string, error) {
func TestDocker(t *testing.T) { func TestDocker(t *testing.T) {
Convey("Docker Should Work", t, func(ctx C) { Convey("Docker Should Work", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
cmdScript := filepath.Join(tmpDir, "cmd.sh") cmdScript := filepath.Join(tmpDir, "cmd.sh")
@ -64,7 +64,7 @@ func TestDocker(t *testing.T) {
echo ${TEST_CONTENT} echo ${TEST_CONTENT}
sleep 20 sleep 20
` `
err = ioutil.WriteFile(cmdScript, []byte(cmdScriptContent), 0755) err = os.WriteFile(cmdScript, []byte(cmdScriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
provider, err := newCmdProvider(c) provider, err := newCmdProvider(c)
@ -125,7 +125,7 @@ sleep 20
So(names, ShouldEqual, "") So(names, ShouldEqual, "")
// check log content // check log content
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput+"\n") So(string(loggedContent), ShouldEqual, expectedOutput+"\n")

11
worker/exec_post_test.go
View File

@ -1,7 +1,6 @@
package worker package worker
import ( import (
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@ -13,7 +12,7 @@ import (
func TestExecPost(t *testing.T) { func TestExecPost(t *testing.T) {
Convey("ExecPost should work", t, func(ctx C) { Convey("ExecPost should work", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
scriptFile := filepath.Join(tmpDir, "cmd.sh") scriptFile := filepath.Join(tmpDir, "cmd.sh")
@ -46,7 +45,7 @@ echo $TUNASYNC_UPSTREAM_URL
echo $TUNASYNC_LOG_FILE echo $TUNASYNC_LOG_FILE
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
go job.Run(managerChan, semaphore) go job.Run(managerChan, semaphore)
@ -64,7 +63,7 @@ echo $TUNASYNC_LOG_FILE
expectedOutput := "success\n" expectedOutput := "success\n"
outputContent, err := ioutil.ReadFile(filepath.Join(provider.WorkingDir(), "exit_status")) outputContent, err := os.ReadFile(filepath.Join(provider.WorkingDir(), "exit_status"))
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(outputContent), ShouldEqual, expectedOutput) So(string(outputContent), ShouldEqual, expectedOutput)
}) })
@ -85,7 +84,7 @@ echo $TUNASYNC_LOG_FILE
exit 1 exit 1
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
go job.Run(managerChan, semaphore) go job.Run(managerChan, semaphore)
@ -105,7 +104,7 @@ exit 1
expectedOutput := "failure\n" expectedOutput := "failure\n"
outputContent, err := ioutil.ReadFile(filepath.Join(provider.WorkingDir(), "exit_status")) outputContent, err := os.ReadFile(filepath.Join(provider.WorkingDir(), "exit_status"))
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(outputContent), ShouldEqual, expectedOutput) So(string(outputContent), ShouldEqual, expectedOutput)
}) })

29
worker/job_test.go
View File

@ -2,7 +2,6 @@ package worker
import ( import (
"fmt" "fmt"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@ -17,7 +16,7 @@ func TestMirrorJob(t *testing.T) {
InitLogger(true, true, false) InitLogger(true, true, false)
Convey("MirrorJob should work", t, func(ctx C) { Convey("MirrorJob should work", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
scriptFile := filepath.Join(tmpDir, "cmd.sh") scriptFile := filepath.Join(tmpDir, "cmd.sh")
@ -58,9 +57,9 @@ func TestMirrorJob(t *testing.T) {
provider.upstreamURL, provider.upstreamURL,
provider.LogFile(), provider.LogFile(),
) )
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
readedScriptContent, err := ioutil.ReadFile(scriptFile) readedScriptContent, err := os.ReadFile(scriptFile)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(readedScriptContent, ShouldResemble, []byte(scriptContent)) So(readedScriptContent, ShouldResemble, []byte(scriptContent))
@ -86,7 +85,7 @@ func TestMirrorJob(t *testing.T) {
So(msg.status, ShouldEqual, Syncing) So(msg.status, ShouldEqual, Syncing)
msg = <-managerChan msg = <-managerChan
So(msg.status, ShouldEqual, Success) So(msg.status, ShouldEqual, Success)
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
job.ctrlChan <- jobStart job.ctrlChan <- jobStart
@ -123,11 +122,11 @@ sleep 3
echo $TUNASYNC_WORKING_DIR echo $TUNASYNC_WORKING_DIR
echo '------' echo '------'
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
hookScriptFile := filepath.Join(tmpDir, "hook.sh") hookScriptFile := filepath.Join(tmpDir, "hook.sh")
err = ioutil.WriteFile(hookScriptFile, []byte(scriptContent), 0755) err = os.WriteFile(hookScriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
h, err := newExecPostHook(provider, execOnFailure, hookScriptFile) h, err := newExecPostHook(provider, execOnFailure, hookScriptFile)
@ -188,7 +187,7 @@ echo $TUNASYNC_WORKING_DIR
sleep 5 sleep 5
echo $TUNASYNC_WORKING_DIR echo $TUNASYNC_WORKING_DIR
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
managerChan := make(chan jobMessage, 10) managerChan := make(chan jobMessage, 10)
@ -213,7 +212,7 @@ echo $TUNASYNC_WORKING_DIR
So(msg.status, ShouldEqual, Failed) So(msg.status, ShouldEqual, Failed)
expectedOutput := fmt.Sprintf("%s\n", provider.WorkingDir()) expectedOutput := fmt.Sprintf("%s\n", provider.WorkingDir())
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
job.ctrlChan <- jobDisable job.ctrlChan <- jobDisable
@ -236,7 +235,7 @@ echo $TUNASYNC_WORKING_DIR
provider.WorkingDir(), provider.WorkingDir(), provider.WorkingDir(), provider.WorkingDir(),
) )
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
job.ctrlChan <- jobDisable job.ctrlChan <- jobDisable
@ -270,7 +269,7 @@ echo $TUNASYNC_WORKING_DIR
provider.WorkingDir(), provider.WorkingDir(), provider.WorkingDir(), provider.WorkingDir(),
) )
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
job.ctrlChan <- jobDisable job.ctrlChan <- jobDisable
@ -326,7 +325,7 @@ echo $TUNASYNC_WORKING_DIR
provider.WorkingDir(), provider.WorkingDir(), provider.WorkingDir(), provider.WorkingDir(),
) )
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
@ -341,7 +340,7 @@ echo $TUNASYNC_WORKING_DIR
sleep 10 sleep 10
echo $TUNASYNC_WORKING_DIR echo $TUNASYNC_WORKING_DIR
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
managerChan := make(chan jobMessage, 10) managerChan := make(chan jobMessage, 10)
@ -364,7 +363,7 @@ echo $TUNASYNC_WORKING_DIR
So(msg.status, ShouldEqual, Failed) So(msg.status, ShouldEqual, Failed)
expectedOutput := fmt.Sprintf("%s\n", provider.WorkingDir()) expectedOutput := fmt.Sprintf("%s\n", provider.WorkingDir())
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
job.ctrlChan <- jobDisable job.ctrlChan <- jobDisable
@ -404,7 +403,7 @@ func TestConcurrentMirrorJobs(t *testing.T) {
InitLogger(true, true, false) InitLogger(true, true, false)
Convey("Concurrent MirrorJobs should work", t, func(ctx C) { Convey("Concurrent MirrorJobs should work", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)

6
worker/loglimit_hook.go
View File

@ -2,7 +2,6 @@ package worker
import ( import (
"fmt" "fmt"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
@ -39,7 +38,7 @@ func (l *logLimiter) preExec() error {
} }
logDir := p.LogDir() logDir := p.LogDir()
files, err := ioutil.ReadDir(logDir) files, err := os.ReadDir(logDir)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
os.MkdirAll(logDir, 0755) os.MkdirAll(logDir, 0755)
@ -50,7 +49,8 @@ func (l *logLimiter) preExec() error {
matchedFiles := []os.FileInfo{} matchedFiles := []os.FileInfo{}
for _, f := range files { for _, f := range files {
if strings.HasPrefix(f.Name(), p.Name()) { if strings.HasPrefix(f.Name(), p.Name()) {
matchedFiles = append(matchedFiles, f) info, _ := f.Info()
matchedFiles = append(matchedFiles, info)
} }
} }

15
worker/loglimit_test.go
View File

@ -2,7 +2,6 @@ package worker
import ( import (
"fmt" "fmt"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@ -14,8 +13,8 @@ import (
func TestLogLimiter(t *testing.T) { func TestLogLimiter(t *testing.T) {
Convey("LogLimiter should work", t, func(ctx C) { Convey("LogLimiter should work", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, _ := os.MkdirTemp("", "tunasync")
tmpLogDir, err := ioutil.TempDir("", "tunasync-log") tmpLogDir, err := os.MkdirTemp("", "tunasync-log")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
defer os.RemoveAll(tmpLogDir) defer os.RemoveAll(tmpLogDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
@ -58,7 +57,7 @@ echo $TUNASYNC_UPSTREAM_URL
echo $TUNASYNC_LOG_FILE echo $TUNASYNC_LOG_FILE
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
go job.Run(managerChan, semaphore) go job.Run(managerChan, semaphore)
@ -86,7 +85,7 @@ echo $TUNASYNC_LOG_FILE
logFile, logFile,
) )
loggedContent, err := ioutil.ReadFile(filepath.Join(provider.LogDir(), "latest")) loggedContent, err := os.ReadFile(filepath.Join(provider.LogDir(), "latest"))
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
}) })
@ -104,7 +103,7 @@ echo $TUNASYNC_LOG_FILE
sleep 5 sleep 5
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
go job.Run(managerChan, semaphore) go job.Run(managerChan, semaphore)
@ -134,10 +133,10 @@ sleep 5
logFile, logFile,
) )
loggedContent, err := ioutil.ReadFile(filepath.Join(provider.LogDir(), "latest")) loggedContent, err := os.ReadFile(filepath.Join(provider.LogDir(), "latest"))
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
loggedContent, err = ioutil.ReadFile(logFile + ".fail") loggedContent, err = os.ReadFile(logFile + ".fail")
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
}) })

63
worker/provider_test.go
View File

@ -2,7 +2,6 @@ package worker
import ( import (
"fmt" "fmt"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
@ -14,7 +13,7 @@ import (
func TestRsyncProvider(t *testing.T) { func TestRsyncProvider(t *testing.T) {
Convey("Rsync Provider should work", t, func() { Convey("Rsync Provider should work", t, func() {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
scriptFile := filepath.Join(tmpDir, "myrsync") scriptFile := filepath.Join(tmpDir, "myrsync")
@ -80,7 +79,7 @@ echo "Total file size: 1.33T bytes"
echo "Done" echo "Done"
exit 0 exit 0
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
targetDir, _ := filepath.EvalSymlinks(provider.WorkingDir()) targetDir, _ := filepath.EvalSymlinks(provider.WorkingDir())
@ -100,7 +99,7 @@ exit 0
err = provider.Run(make(chan empty, 1)) err = provider.Run(make(chan empty, 1))
So(err, ShouldBeNil) So(err, ShouldBeNil)
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
// fmt.Println(string(loggedContent)) // fmt.Println(string(loggedContent))
@ -109,7 +108,7 @@ exit 0
}) })
Convey("If the rsync program fails", t, func() { Convey("If the rsync program fails", t, func() {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
tmpFile := filepath.Join(tmpDir, "log_file") tmpFile := filepath.Join(tmpDir, "log_file")
@ -131,7 +130,7 @@ exit 0
err = provider.Run(make(chan empty, 1)) err = provider.Run(make(chan empty, 1))
So(err, ShouldNotBeNil) So(err, ShouldNotBeNil)
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldContainSubstring, "Syntax or usage error") So(string(loggedContent), ShouldContainSubstring, "Syntax or usage error")
}) })
@ -140,7 +139,7 @@ exit 0
func TestRsyncProviderWithAuthentication(t *testing.T) { func TestRsyncProviderWithAuthentication(t *testing.T) {
Convey("Rsync Provider with password should work", t, func() { Convey("Rsync Provider with password should work", t, func() {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
scriptFile := filepath.Join(tmpDir, "myrsync") scriptFile := filepath.Join(tmpDir, "myrsync")
@ -180,7 +179,7 @@ sleep 1
echo "Done" echo "Done"
exit 0 exit 0
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
targetDir, _ := filepath.EvalSymlinks(provider.WorkingDir()) targetDir, _ := filepath.EvalSymlinks(provider.WorkingDir())
@ -200,7 +199,7 @@ exit 0
err = provider.Run(make(chan empty, 1)) err = provider.Run(make(chan empty, 1))
So(err, ShouldBeNil) So(err, ShouldBeNil)
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
// fmt.Println(string(loggedContent)) // fmt.Println(string(loggedContent))
@ -211,7 +210,7 @@ exit 0
func TestRsyncProviderWithOverriddenOptions(t *testing.T) { func TestRsyncProviderWithOverriddenOptions(t *testing.T) {
Convey("Rsync Provider with overridden options should work", t, func() { Convey("Rsync Provider with overridden options should work", t, func() {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
scriptFile := filepath.Join(tmpDir, "myrsync") scriptFile := filepath.Join(tmpDir, "myrsync")
@ -248,7 +247,7 @@ sleep 1
echo "Done" echo "Done"
exit 0 exit 0
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
targetDir, _ := filepath.EvalSymlinks(provider.WorkingDir()) targetDir, _ := filepath.EvalSymlinks(provider.WorkingDir())
@ -263,7 +262,7 @@ exit 0
err = provider.Run(make(chan empty, 1)) err = provider.Run(make(chan empty, 1))
So(err, ShouldBeNil) So(err, ShouldBeNil)
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
// fmt.Println(string(loggedContent)) // fmt.Println(string(loggedContent))
@ -274,7 +273,7 @@ exit 0
func TestRsyncProviderWithDocker(t *testing.T) { func TestRsyncProviderWithDocker(t *testing.T) {
Convey("Rsync in Docker should work", t, func() { Convey("Rsync in Docker should work", t, func() {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
scriptFile := filepath.Join(tmpDir, "myrsync") scriptFile := filepath.Join(tmpDir, "myrsync")
@ -323,9 +322,9 @@ fi
shift shift
done done
` `
err = ioutil.WriteFile(scriptFile, []byte(cmdScriptContent), 0755) err = os.WriteFile(scriptFile, []byte(cmdScriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
err = ioutil.WriteFile(excludeFile, []byte("__some_pattern"), 0755) err = os.WriteFile(excludeFile, []byte("__some_pattern"), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
for _, hook := range provider.Hooks() { for _, hook := range provider.Hooks() {
@ -338,7 +337,7 @@ done
err = hook.postExec() err = hook.postExec()
So(err, ShouldBeNil) So(err, ShouldBeNil)
} }
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, "__some_pattern") So(string(loggedContent), ShouldEqual, "__some_pattern")
}) })
@ -346,7 +345,7 @@ done
func TestCmdProvider(t *testing.T) { func TestCmdProvider(t *testing.T) {
Convey("Command Provider should work", t, func(ctx C) { Convey("Command Provider should work", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
scriptFile := filepath.Join(tmpDir, "cmd.sh") scriptFile := filepath.Join(tmpDir, "cmd.sh")
@ -391,25 +390,25 @@ echo $AOSP_REPO_BIN
provider.LogFile(), provider.LogFile(),
"/usr/local/bin/repo", "/usr/local/bin/repo",
) )
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
readedScriptContent, err := ioutil.ReadFile(scriptFile) readedScriptContent, err := os.ReadFile(scriptFile)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(readedScriptContent, ShouldResemble, []byte(scriptContent)) So(readedScriptContent, ShouldResemble, []byte(scriptContent))
err = provider.Run(make(chan empty, 1)) err = provider.Run(make(chan empty, 1))
So(err, ShouldBeNil) So(err, ShouldBeNil)
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
}) })
Convey("If a command fails", func() { Convey("If a command fails", func() {
scriptContent := `exit 1` scriptContent := `exit 1`
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
readedScriptContent, err := ioutil.ReadFile(scriptFile) readedScriptContent, err := os.ReadFile(scriptFile)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(readedScriptContent, ShouldResemble, []byte(scriptContent)) So(readedScriptContent, ShouldResemble, []byte(scriptContent))
@ -422,7 +421,7 @@ echo $AOSP_REPO_BIN
scriptContent := `#!/bin/bash scriptContent := `#!/bin/bash
sleep 10 sleep 10
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
started := make(chan empty, 1) started := make(chan empty, 1)
@ -440,7 +439,7 @@ sleep 10
}) })
}) })
Convey("Command Provider without log file should work", t, func(ctx C) { Convey("Command Provider without log file should work", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
@ -474,7 +473,7 @@ sleep 10
}) })
}) })
Convey("Command Provider with RegExprs should work", t, func(ctx C) { Convey("Command Provider with RegExprs should work", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
tmpFile := filepath.Join(tmpDir, "log_file") tmpFile := filepath.Join(tmpDir, "log_file")
@ -557,7 +556,7 @@ sleep 10
func TestTwoStageRsyncProvider(t *testing.T) { func TestTwoStageRsyncProvider(t *testing.T) {
Convey("TwoStageRsync Provider should work", t, func(ctx C) { Convey("TwoStageRsync Provider should work", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
scriptFile := filepath.Join(tmpDir, "myrsync") scriptFile := filepath.Join(tmpDir, "myrsync")
@ -597,7 +596,7 @@ sleep 1
echo "Done" echo "Done"
exit 0 exit 0
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
err = provider.Run(make(chan empty, 2)) err = provider.Run(make(chan empty, 2))
@ -627,7 +626,7 @@ exit 0
), ),
) )
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldEqual, expectedOutput) So(string(loggedContent), ShouldEqual, expectedOutput)
// fmt.Println(string(loggedContent)) // fmt.Println(string(loggedContent))
@ -639,7 +638,7 @@ echo $@
sleep 10 sleep 10
exit 0 exit 0
` `
err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) err = os.WriteFile(scriptFile, []byte(scriptContent), 0755)
So(err, ShouldBeNil) So(err, ShouldBeNil)
started := make(chan empty, 2) started := make(chan empty, 2)
@ -661,7 +660,7 @@ exit 0
provider.excludeFile, provider.upstreamURL, provider.WorkingDir(), provider.excludeFile, provider.upstreamURL, provider.WorkingDir(),
) )
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldStartWith, expectedOutput) So(string(loggedContent), ShouldStartWith, expectedOutput)
// fmt.Println(string(loggedContent)) // fmt.Println(string(loggedContent))
@ -669,7 +668,7 @@ exit 0
}) })
Convey("If the rsync program fails", t, func(ctx C) { Convey("If the rsync program fails", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
defer os.RemoveAll(tmpDir) defer os.RemoveAll(tmpDir)
So(err, ShouldBeNil) So(err, ShouldBeNil)
tmpFile := filepath.Join(tmpDir, "log_file") tmpFile := filepath.Join(tmpDir, "log_file")
@ -691,7 +690,7 @@ exit 0
err = provider.Run(make(chan empty, 2)) err = provider.Run(make(chan empty, 2))
So(err, ShouldNotBeNil) So(err, ShouldNotBeNil)
loggedContent, err := ioutil.ReadFile(provider.LogFile()) loggedContent, err := os.ReadFile(provider.LogFile())
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(string(loggedContent), ShouldContainSubstring, "Error in socket I/O") So(string(loggedContent), ShouldContainSubstring, "Error in socket I/O")

2
worker/worker.go
View File

@ -317,7 +317,7 @@ func (w *Worker) runSchedule() {
schedInfo := w.schedule.GetJobs() schedInfo := w.schedule.GetJobs()
w.updateSchedInfo(schedInfo) w.updateSchedInfo(schedInfo)
tick := time.Tick(5 * time.Second) tick := time.NewTicker(5 * time.Second).C
for { for {
select { select {
case jobMsg := <-w.managerChan: case jobMsg := <-w.managerChan:

3
worker/zfs_hook_test.go
View File

@ -1,7 +1,6 @@
package worker package worker
import ( import (
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@ -13,7 +12,7 @@ import (
func TestZFSHook(t *testing.T) { func TestZFSHook(t *testing.T) {
Convey("ZFS Hook should work", t, func(ctx C) { Convey("ZFS Hook should work", t, func(ctx C) {
tmpDir, err := ioutil.TempDir("", "tunasync") tmpDir, err := os.MkdirTemp("", "tunasync")
tmpFile := filepath.Join(tmpDir, "log_file") tmpFile := filepath.Join(tmpDir, "log_file")
c := cmdConfig{ c := cmdConfig{