From 33611cee8e163c414b13b05b561f8277fb0add6f Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 24 Mar 2016 22:23:41 +0800 Subject: [PATCH 01/66] docs: new desgin --- README.md | 49 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f3bb7dd..8dba249 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,45 @@ tunasync ======== +## Design + +``` +# Architecture + +- Manager: Centural instance on status and job management +- Worker: Runs mirror jobs + + ++----------+ +---+ worker configs +---+ +----------+ +----------+ +| Status | | |+-----------------> | w +--->| mirror +---->| mirror | +| Manager | | | | o | | config | | provider | ++----------+ | W | start/stop job | r | +----------+ +----+-----+ + | E |+-----------------> | k | | ++----------+ | B | | e | +------------+ | +| Job | | | update status | r |<------+ mirror job |<----+ +|Controller| | | <-----------------+| | +------------+ ++----------+ +---+ +---+ + + +# Job Run Process + ++-----------+ +-----------+ +-------------+ +--------------+ +| pre-job +--+->| job run +--->| post-job +-+-->| post-success | ++-----------+ ^ +-----------+ +-------------+ | +--------------+ + | | + | +-----------------+ | + +------+ post-fail |<---------+ + +-----------------+ +``` + ## TODO -- [ ] use context manager to handle job contexts -- [x] Hooks need "before_exec", "after_exec" -- [x] implement `tunasynctl tail` and `tunasynctl log` or equivalent feature -- [x] status file - - [ ] mirror size - - [x] upstream -- [x] btrfs backend (create snapshot before syncing) -- [x] add mirror job online -- [x] use toml as configuration +- [ ] split to `tunasync-manager` and `tunasync-worker` instances + - use HTTP as communication protocol +- Web frontend for `tunasync-manager` + - [ ] start/stop/restart job + - [ ] enable/disable mirror + - [ ] view log +- [ ] config file structure + - [ ] support multi-file configuration (`/etc/tunasync.d/mirror-enabled/*.conf`) + From d735b1eb71377a4e0fbc0eeb93221045259ce273 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sat, 2 Apr 2016 16:36:40 +0800 Subject: [PATCH 02/66] docs: add doc to generate self-signed certs --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/README.md b/README.md index 8dba249..8998a4b 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,60 @@ tunasync - [ ] config file structure - [ ] support multi-file configuration (`/etc/tunasync.d/mirror-enabled/*.conf`) +## Generate Self-Signed Certificate + +Fisrt, create root CA + +``` +openssl genrsa -out rootCA.key 2048 +openssl req -x509 -new -nodes -key rootCA.key -days 365 -out rootCA.crt +``` + +Create host key + +``` +openssl genrsa -out host.key 2048 +``` + +Now create CSR, before that, write a `req.cnf` + +``` +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req + +[req_distinguished_name] +countryName = Country Name (2 letter code) +countryName_default = CN +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = BJ +localityName = Locality Name (eg, city) +localityName_default = Beijing +organizationalUnitName = Organizational Unit Name (eg, section) +organizationalUnitName_default = TUNA +commonName = Common Name (server FQDN or domain name) +commonName_default = +commonName_max = 64 + +[v3_req] +# Extensions to add to a certificate request +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = +DNS.2 = +``` + +Substitute `` with your server's FQDN, then run + +``` +openssl req -new -key host.key -out host.csr -config req.cnf +``` + +Finally generate and sign host cert with root CA + +``` +openssl x509 -req -in host.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out host.crt -days 365 -extensions v3_req -extfile req.cnf +``` From 23fd9681b39021eb8ac73faa449e3435d7fdfc02 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sat, 2 Apr 2016 17:02:53 +0800 Subject: [PATCH 03/66] docs: ideas --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8998a4b..3a68297 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ tunasync ======== +## Ideas + +- use [etcd](https://github.com/coreos/etcd) to store configurations and state variables + ## Design ``` From 350767e501b2e0bd9e60fac70ff5f89cb3705e43 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 10 Apr 2016 21:53:40 +0800 Subject: [PATCH 04/66] feature(manager): Manager server logger and config --- internal/logger.go | 28 ++++++++ manager/common.go | 7 ++ manager/config.go | 63 ++++++++++++++++++ manager/config_test.go | 141 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 239 insertions(+) create mode 100644 internal/logger.go create mode 100644 manager/common.go create mode 100644 manager/config.go create mode 100644 manager/config_test.go diff --git a/internal/logger.go b/internal/logger.go new file mode 100644 index 0000000..bf1a978 --- /dev/null +++ b/internal/logger.go @@ -0,0 +1,28 @@ +package internal + +import ( + "os" + + "gopkg.in/op/go-logging.v1" +) + +// InitLogger initilizes logging format and level +func InitLogger(verbose, debug, withSystemd bool) { + var fmtString string + if withSystemd { + fmtString = "\r[%{level:.6s}] %{message}" + } else { + fmtString = "\r%{color}[%{time:06-01-02 15:04:05}][%{level:.6s}]%{color:reset} %{message}" + } + format := logging.MustStringFormatter(fmtString) + logging.SetFormatter(format) + logging.SetBackend(logging.NewLogBackend(os.Stdout, "", 0)) + + if debug { + logging.SetLevel(logging.DEBUG, "tunasync") + } else if verbose { + logging.SetLevel(logging.INFO, "tunasync") + } else { + logging.SetLevel(logging.NOTICE, "tunasync") + } +} diff --git a/manager/common.go b/manager/common.go new file mode 100644 index 0000000..2c6d88a --- /dev/null +++ b/manager/common.go @@ -0,0 +1,7 @@ +package manager + +import ( + "gopkg.in/op/go-logging.v1" +) + +var logger = logging.MustGetLogger("tunasync") diff --git a/manager/config.go b/manager/config.go new file mode 100644 index 0000000..54d8568 --- /dev/null +++ b/manager/config.go @@ -0,0 +1,63 @@ +package manager + +import ( + "github.com/BurntSushi/toml" + "github.com/codegangsta/cli" +) + +// A Config is the top-level toml-serializaible config struct +type Config struct { + Debug bool `toml:"debug"` + Server ServerConfig `toml:"server"` + Files FileConfig `toml:"files"` +} + +// A ServerConfig represents the configuration for HTTP server +type ServerConfig struct { + Addr string `toml:"addr"` + Port int `toml:"port"` + SSLCert string `toml:"ssl_cert"` + SSLKey string `toml:"ssl_key"` +} + +// A FileConfig contains paths to special files +type FileConfig struct { + StatusFile string `toml:"status_file"` + DBFile string `toml:"db_file"` +} + +func loadConfig(cfgFile string, c *cli.Context) (*Config, error) { + + cfg := new(Config) + cfg.Server.Addr = "127.0.0.1" + cfg.Server.Port = 14242 + cfg.Debug = false + cfg.Files.StatusFile = "/var/lib/tunasync/tunasync.json" + cfg.Files.DBFile = "/var/lib/tunasync/tunasync.db" + + if cfgFile != "" { + if _, err := toml.DecodeFile(cfgFile, cfg); err != nil { + logger.Error(err.Error()) + return nil, err + } + } + + if c.String("addr") != "" { + cfg.Server.Addr = c.String("addr") + } + if c.Int("port") > 0 { + cfg.Server.Port = c.Int("port") + } + if c.String("cert") != "" && c.String("key") != "" { + cfg.Server.SSLCert = c.String("cert") + cfg.Server.SSLKey = c.String("key") + } + if c.String("status-file") != "" { + cfg.Files.StatusFile = c.String("status-file") + } + if c.String("db-file") != "" { + cfg.Files.DBFile = c.String("db-file") + } + + return cfg, nil +} diff --git a/manager/config_test.go b/manager/config_test.go new file mode 100644 index 0000000..4a81347 --- /dev/null +++ b/manager/config_test.go @@ -0,0 +1,141 @@ +package manager + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/BurntSushi/toml" + "github.com/codegangsta/cli" + . "github.com/smartystreets/goconvey/convey" +) + +func TestConfig(t *testing.T) { + var cfgBlob = ` + debug = true + [server] + addr = "0.0.0.0" + port = 5000 + + [files] + status_file = "/tmp/tunasync.json" + db_file = "/var/lib/tunasync/tunasync.db" + ` + + Convey("toml decoding should work", t, func() { + + var conf Config + _, err := toml.Decode(cfgBlob, &conf) + ShouldEqual(err, nil) + ShouldEqual(conf.Server.Addr, "0.0.0.0") + ShouldEqual(conf.Server.Port, 5000) + ShouldEqual(conf.Files.StatusFile, "/tmp/tunasync.json") + ShouldEqual(conf.Files.DBFile, "/var/lib/tunasync/tunasync.db") + }) + + Convey("load Config should work", t, func() { + Convey("create config file & cli context", func() { + tmpfile, err := ioutil.TempFile("", "tunasync") + So(err, ShouldEqual, nil) + defer os.Remove(tmpfile.Name()) + + err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644) + So(err, ShouldEqual, nil) + defer tmpfile.Close() + + app := cli.NewApp() + app.Flags = []cli.Flag{ + cli.StringFlag{ + Name: "config, c", + }, + cli.StringFlag{ + Name: "addr", + }, + cli.IntFlag{ + Name: "port", + }, + cli.StringFlag{ + Name: "cert", + }, + cli.StringFlag{ + Name: "key", + }, + cli.StringFlag{ + Name: "status-file", + }, + cli.StringFlag{ + Name: "db-file", + }, + } + Convey("when giving no config options", func() { + app.Action = func(c *cli.Context) { + cfgFile := c.String("config") + cfg, err := loadConfig(cfgFile, c) + So(err, ShouldEqual, nil) + So(cfg.Server.Addr, ShouldEqual, "127.0.0.1") + } + args := strings.Split("cmd", " ") + app.Run(args) + }) + Convey("when giving config options", func() { + app.Action = func(c *cli.Context) { + cfgFile := c.String("config") + So(cfgFile, ShouldEqual, tmpfile.Name()) + conf, err := loadConfig(cfgFile, c) + So(err, ShouldEqual, nil) + So(conf.Server.Addr, ShouldEqual, "0.0.0.0") + So(conf.Server.Port, ShouldEqual, 5000) + So(conf.Files.StatusFile, ShouldEqual, "/tmp/tunasync.json") + So(conf.Files.DBFile, ShouldEqual, "/var/lib/tunasync/tunasync.db") + + } + cmd := fmt.Sprintf("cmd -c %s", tmpfile.Name()) + args := strings.Split(cmd, " ") + app.Run(args) + }) + Convey("when giving cli options", func() { + app.Action = func(c *cli.Context) { + cfgFile := c.String("config") + So(cfgFile, ShouldEqual, "") + conf, err := loadConfig(cfgFile, c) + So(err, ShouldEqual, nil) + So(conf.Server.Addr, ShouldEqual, "0.0.0.0") + So(conf.Server.Port, ShouldEqual, 5001) + So(conf.Server.SSLCert, ShouldEqual, "/ssl.cert") + So(conf.Server.SSLKey, ShouldEqual, "/ssl.key") + So(conf.Files.StatusFile, ShouldEqual, "/tunasync.json") + So(conf.Files.DBFile, ShouldEqual, "/tunasync.db") + + } + args := strings.Split( + "cmd --addr=0.0.0.0 --port=5001 --cert=/ssl.cert --key /ssl.key --status-file=/tunasync.json --db-file=/tunasync.db", + " ", + ) + app.Run(args) + }) + Convey("when giving both config and cli options", func() { + app.Action = func(c *cli.Context) { + cfgFile := c.String("config") + So(cfgFile, ShouldEqual, tmpfile.Name()) + conf, err := loadConfig(cfgFile, c) + So(err, ShouldEqual, nil) + So(conf.Server.Addr, ShouldEqual, "0.0.0.0") + So(conf.Server.Port, ShouldEqual, 5000) + So(conf.Server.SSLCert, ShouldEqual, "/ssl.cert") + So(conf.Server.SSLKey, ShouldEqual, "/ssl.key") + So(conf.Files.StatusFile, ShouldEqual, "/tunasync.json") + So(conf.Files.DBFile, ShouldEqual, "/tunasync.db") + + } + cmd := fmt.Sprintf( + "cmd -c %s --cert=/ssl.cert --key /ssl.key --status-file=/tunasync.json --db-file=/tunasync.db", + tmpfile.Name(), + ) + args := strings.Split(cmd, " ") + app.Run(args) + }) + }) + }) +} From ed896b16c1ee2bf4c8f446221e729bbe683df715 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 10 Apr 2016 22:28:34 +0800 Subject: [PATCH 05/66] feature(manager): skeleton of status API --- manager/config.go | 2 ++ manager/server.go | 36 ++++++++++++++++++++++++++++++++++++ manager/server_test.go | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 manager/server.go create mode 100644 manager/server_test.go diff --git a/manager/config.go b/manager/config.go index 54d8568..9a422f7 100644 --- a/manager/config.go +++ b/manager/config.go @@ -24,6 +24,8 @@ type ServerConfig struct { type FileConfig struct { StatusFile string `toml:"status_file"` DBFile string `toml:"db_file"` + // used to connect to worker + CACert string `toml:"ca_cert"` } func loadConfig(cfgFile string, c *cli.Context) (*Config, error) { diff --git a/manager/server.go b/manager/server.go new file mode 100644 index 0000000..3c41889 --- /dev/null +++ b/manager/server.go @@ -0,0 +1,36 @@ +package manager + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +type worker struct { + // worker name + name string + // url to connect to worker + url string + // session token + token string +} + +func makeHTTPServer(debug bool) *gin.Engine { + if !debug { + gin.SetMode(gin.ReleaseMode) + } + r := gin.Default() + r.GET("/ping", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"msg": "pong"}) + }) + // List jobs, status page + r.GET("/jobs", func(c *gin.Context) {}) + // worker online + r.POST("/workers/:name", func(c *gin.Context) {}) + // post job list + r.POST("/workers/:name/jobs", func(c *gin.Context) {}) + // post job status + r.POST("/workers/:name/jobs/:job", func(c *gin.Context) {}) + + return r +} diff --git a/manager/server_test.go b/manager/server_test.go new file mode 100644 index 0000000..c6cb0e7 --- /dev/null +++ b/manager/server_test.go @@ -0,0 +1,37 @@ +package manager + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestHTTPServer(t *testing.T) { + Convey("HTTP server should work", t, func() { + s := makeHTTPServer(false) + So(s, ShouldNotBeNil) + port := rand.Intn(10000) + 20000 + go func() { + s.Run(fmt.Sprintf("127.0.0.1:%d", port)) + }() + time.Sleep(50 * time.Microsecond) + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/ping", port)) + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + So(resp.Header.Get("Content-Type"), ShouldEqual, "application/json; charset=utf-8") + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + So(err, ShouldBeNil) + var p map[string]string + err = json.Unmarshal(body, &p) + So(err, ShouldBeNil) + So(p["msg"], ShouldEqual, "pong") + }) + +} From 66df20cb1dc61e565fff6730e63c7980a026930b Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 10 Apr 2016 22:40:48 +0800 Subject: [PATCH 06/66] chore: travis CI and coverall integration --- .testandcover.bash | 31 +++++++++++++++++++++++++++++++ .testpackages.txt | 2 ++ .travis.yml | 16 ++++++++++++++++ README.md | 9 +++++---- 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100755 .testandcover.bash create mode 100644 .testpackages.txt create mode 100644 .travis.yml diff --git a/.testandcover.bash b/.testandcover.bash new file mode 100755 index 0000000..8f88b18 --- /dev/null +++ b/.testandcover.bash @@ -0,0 +1,31 @@ +#!/bin/bash + + +function die() { + echo $* + exit 1 +} + +export GOPATH=`pwd`:$GOPATH + +# Initialize profile.cov +echo "mode: count" > profile.cov + +# Initialize error tracking +ERROR="" + +# Test each package and append coverage profile info to profile.cov +for pkg in `cat .testpackages.txt` +do + #$HOME/gopath/bin/ + go test -v -covermode=count -coverprofile=profile_tmp.cov $pkg || ERROR="Error testing $pkg" + + [ -f profile_tmp.cov ] && { + tail -n +2 profile_tmp.cov >> profile.cov || die "Unable to append coverage for $pkg" + } +done + +if [ ! -z "$ERROR" ] +then + die "Encountered error, last error was: $ERROR" +fi diff --git a/.testpackages.txt b/.testpackages.txt new file mode 100644 index 0000000..ba04577 --- /dev/null +++ b/.testpackages.txt @@ -0,0 +1,2 @@ +github.com/tuna/tunasync/internal +github.com/tuna/tunasync/manager diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d6a4831 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,16 @@ +language: go +go: + - 1.6 + +before_install: + - go get golang.org/x/tools/cmd/cover + - go get -v github.com/mattn/goveralls + +os: + - linux + +script: + - ./.testandcover.bash + +after_success: + - goveralls -coverprofile=profile.cov -service=travis-ci diff --git a/README.md b/README.md index 3a68297..485cba9 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ tunasync ======== -## Ideas - -- use [etcd](https://github.com/coreos/etcd) to store configurations and state variables +[![Build Status](https://travis-ci.org/tuna/tunasync.svg?branch=dev)](https://travis-ci.org/tuna/tunasync) +[![Coverage Status](https://coveralls.io/repos/github/tuna/tunasync/badge.svg?branch=dev)](https://coveralls.io/github/tuna/tunasync?branch=dev) ## Design @@ -39,7 +38,9 @@ tunasync ## TODO - [ ] split to `tunasync-manager` and `tunasync-worker` instances - - use HTTP as communication protocol + - [ ] use HTTP as communication protocol + - [ ] implement manager as status server first, and use python worker + - [ ] implement go worker - Web frontend for `tunasync-manager` - [ ] start/stop/restart job - [ ] enable/disable mirror From 3d38e413ce8cd0ee2fa78571e1cf72972cf2c977 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 15 Apr 2016 00:47:50 +0800 Subject: [PATCH 07/66] feature: syncing status --- .testpackages.txt | 1 + internal/status/status.go | 144 +++++++++++++++++++++++++++++++++ internal/status/status_test.go | 39 +++++++++ 3 files changed, 184 insertions(+) create mode 100644 internal/status/status.go create mode 100644 internal/status/status_test.go diff --git a/.testpackages.txt b/.testpackages.txt index ba04577..9e724cc 100644 --- a/.testpackages.txt +++ b/.testpackages.txt @@ -1,2 +1,3 @@ github.com/tuna/tunasync/internal +github.com/tuna/tunasync/internal/status github.com/tuna/tunasync/manager diff --git a/internal/status/status.go b/internal/status/status.go new file mode 100644 index 0000000..dfab49c --- /dev/null +++ b/internal/status/status.go @@ -0,0 +1,144 @@ +package status + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +type syncStatus uint8 + +const ( + None syncStatus = iota + Failed + Success + Syncing + PreSyncing + Paused + Disabled +) + +type MirrorStatus struct { + Name string + Status syncStatus + LastUpdate time.Time + Upstream string + Size string // approximate size +} + +func (s MirrorStatus) MarshalJSON() ([]byte, error) { + m := map[string]interface{}{ + "name": s.Name, + "status": s.Status, + "last_update": s.LastUpdate.Format("2006-01-02 15:04:05"), + "last_update_ts": fmt.Sprintf("%d", s.LastUpdate.Unix()), + "size": s.Size, + "upstream": s.Upstream, + } + return json.Marshal(m) +} + +func (s *MirrorStatus) UnmarshalJSON(v []byte) error { + var m map[string]interface{} + + err := json.Unmarshal(v, &m) + if err != nil { + return err + } + + if name, ok := m["name"]; ok { + if s.Name, ok = name.(string); !ok { + return errors.New("name should be a string") + } + } else { + return errors.New("key `name` does not exist in the json") + } + if upstream, ok := m["upstream"]; ok { + if s.Upstream, ok = upstream.(string); !ok { + return errors.New("upstream should be a string") + } + } else { + return errors.New("key `upstream` does not exist in the json") + } + if size, ok := m["size"]; ok { + if s.Size, ok = size.(string); !ok { + return errors.New("size should be a string") + } + } else { + return errors.New("key `size` does not exist in the json") + } + // tricky: status + if status, ok := m["status"]; ok { + if ss, ok := status.(string); ok { + err := json.Unmarshal([]byte(`"`+ss+`"`), &(s.Status)) + if err != nil { + return err + } + } else { + return errors.New("status should be a string") + } + } else { + return errors.New("key `status` does not exist in the json") + } + // tricky: last update + if lastUpdate, ok := m["last_update_ts"]; ok { + if sts, ok := lastUpdate.(string); ok { + ts, err := strconv.Atoi(sts) + if err != nil { + return fmt.Errorf("last_update_ts should be a interger, got: %s", sts) + } + s.LastUpdate = time.Unix(int64(ts), 0) + } else { + return fmt.Errorf("last_update_ts should be a string of integer, got: %s", lastUpdate) + } + } else { + return errors.New("key `last_update_ts` does not exist in the json") + } + return nil +} + +func (s syncStatus) MarshalJSON() ([]byte, error) { + var strStatus string + switch s { + case None: + strStatus = "none" + case Success: + strStatus = "success" + case Syncing: + strStatus = "syncing" + case PreSyncing: + strStatus = "pre-syncing" + case Paused: + strStatus = "paused" + case Disabled: + strStatus = "disabled" + default: + return []byte{}, errors.New("Invalid status value") + } + + return json.Marshal(strStatus) +} + +func (s *syncStatus) UnmarshalJSON(v []byte) error { + sv := strings.Trim(string(v), `"`) + switch sv { + case "none": + *s = None + case "success": + *s = Success + case "syncing": + *s = Syncing + case "pre-syncing": + *s = PreSyncing + case "paused": + *s = Paused + case "disabled": + *s = Disabled + default: + return fmt.Errorf("Invalid status value: %s", string(v)) + } + return nil +} diff --git a/internal/status/status_test.go b/internal/status/status_test.go new file mode 100644 index 0000000..578a115 --- /dev/null +++ b/internal/status/status_test.go @@ -0,0 +1,39 @@ +package status + +import ( + "encoding/json" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestStatus(t *testing.T) { + Convey("status json ser-de should work", t, func() { + tz := "Asia/Shanghai" + loc, err := time.LoadLocation(tz) + So(err, ShouldBeNil) + + m := MirrorStatus{ + Name: "tunalinux", + Status: Success, + LastUpdate: time.Date(2016, time.April, 16, 23, 8, 10, 0, loc), + Size: "5GB", + Upstream: "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/", + } + + b, err := json.Marshal(m) + So(err, ShouldBeNil) + // fmt.Println(string(b)) + var m2 MirrorStatus + err = json.Unmarshal(b, &m2) + So(err, ShouldBeNil) + // fmt.Printf("%#v", m2) + So(m2.Name, ShouldEqual, m.Name) + So(m2.Status, ShouldEqual, m.Status) + So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix()) + So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) + So(m2.Size, ShouldEqual, m.Size) + So(m2.Upstream, ShouldEqual, m.Upstream) + }) +} From 96f38363eac30aff71b62972f23553f553f9684f Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 21 Apr 2016 19:38:37 +0800 Subject: [PATCH 08/66] refactor: remove part of unneeded files --- examples/shell_provider.sh | 7 --- examples/tunasync.conf | 75 ---------------------------- requirements.txt | 3 -- systemd/tunasync-snapshot-gc.service | 11 ---- systemd/tunasync-snapshot-gc.timer | 8 --- systemd/tunasync.service | 13 ----- tunasync.py | 28 ----------- tunasync_snapshot_gc.py | 43 ---------------- tunasynctl.py | 64 ------------------------ 9 files changed, 252 deletions(-) delete mode 100755 examples/shell_provider.sh delete mode 100644 examples/tunasync.conf delete mode 100644 requirements.txt delete mode 100644 systemd/tunasync-snapshot-gc.service delete mode 100644 systemd/tunasync-snapshot-gc.timer delete mode 100644 systemd/tunasync.service delete mode 100644 tunasync.py delete mode 100644 tunasync_snapshot_gc.py delete mode 100755 tunasynctl.py diff --git a/examples/shell_provider.sh b/examples/shell_provider.sh deleted file mode 100755 index 4ffbd4b..0000000 --- a/examples/shell_provider.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -echo $TUNASYNC_WORKING_DIR -echo $TUNASYNC_LOG_FILE -echo $TUNASYNC_UPSTREAM_URL -echo $REPO -sleep 5 -exit 1 diff --git a/examples/tunasync.conf b/examples/tunasync.conf deleted file mode 100644 index 6db68c5..0000000 --- a/examples/tunasync.conf +++ /dev/null @@ -1,75 +0,0 @@ -[global] -log_dir = "/var/log/tunasync" -# mirror_root = /srv/mirror_disk -mirror_root = "/mnt/sdb1/mirror" -use_btrfs = false -local_dir = "{mirror_root}/_working/{mirror_name}/" -status_file = "/tmp/tunasync.json" -# maximum numbers of running jobs -concurrent = 2 -# interval in minutes -interval = 1 -max_retry = 2 -ctrl_addr = "/tmp/tunasync.sock" - -[btrfs] -service_dir = "{mirror_root}/_current/{mirror_name}" -working_dir = "{mirror_root}/_working/{mirror_name}" -gc_root = "{mirror_root}/_garbage/" -gc_dir = "{mirror_root}/_garbage/_gc_{mirror_name}_{{timestamp}}" - -# [[mirrors]] -# name = "archlinux" -# provider = "rsync" -# upstream = "rsync://mirror.us.leaseweb.net/archlinux/" -# log_file = "/tmp/archlinux-{date}.log" -# use_ipv6 = true - -[[mirrors]] -name = "arch1" -provider = "shell" -command = "sleep 10" -local_dir = "/mnt/sdb1/mirror/archlinux/current/" -# log_file = "/dev/null" -exec_post_sync = "/bin/bash -c 'date --utc \"+%s\" > ${TUNASYNC_WORKING_DIR}/.timestamp'" - -[[mirrors]] -name = "arch2" -provider = "shell" -command = "sleep 20" -local_dir = "/mnt/sdb1/mirror/archlinux/current/" -# log_file = "/dev/null" - - -[[mirrors]] -name = "arch3" -provider = "two-stage-rsync" -stage1_profile = "debian" -upstream = "/tmp/rsync_test/src/" -local_dir = "/tmp/rsync_test/dst/" -log_file = "/tmp/rsync_test/log" -# log_file = "/dev/null" -no_delay = true - -[[mirrors]] -name = "arch4" -provider = "shell" -command = "./shell_provider.sh" -upstream = "https://pypi.python.org/" -# log_file = "/tmp/arch4-{date}.log" -use_btrfs = false - # set environment varialbes - [mirrors.env] - REPO = "/usr/local/bin/repo" - -[[mirrors]] -name = "arch5" -provider = "shell" -command = "./shell_provider.sh" -upstream = "https://pypi.python.org/" -# log_file = "/tmp/arch4-{date}.log" -use_btrfs = false - [mirrors.env] - REPO = "/usr/local/bin/repo2" - -# vim: ft=toml ts=2 sts=2 sw=2 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 9787eee..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -setproctitle==1.1.9 -sh==1.11 -toml==0.9.1 diff --git a/systemd/tunasync-snapshot-gc.service b/systemd/tunasync-snapshot-gc.service deleted file mode 100644 index 0dc3f91..0000000 --- a/systemd/tunasync-snapshot-gc.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Delete garbage subvolumes generated by tunasync -Requires = network.target -After = network.target - -[Service] -Type=oneshot -ExecStart=/home/tuna/.virtualenvs/tunasync/bin/python -u /home/tuna/tunasync/tunasync_snapshot_gc.py -c /etc/tunasync.ini - -[Install] -WantedBy = multi-user.target diff --git a/systemd/tunasync-snapshot-gc.timer b/systemd/tunasync-snapshot-gc.timer deleted file mode 100644 index 674e6a1..0000000 --- a/systemd/tunasync-snapshot-gc.timer +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=TUNAsync GC every 10 minutes - -[Timer] -OnUnitActiveSec=10min - -[Install] -WantedBy=multi-user.target diff --git a/systemd/tunasync.service b/systemd/tunasync.service deleted file mode 100644 index df5e902..0000000 --- a/systemd/tunasync.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description = TUNA mirrors sync daemon -Requires = network.target -After = network.target - -[Service] -ExecStart = /home/tuna/.virtualenvs/tunasync/bin/python -u /home/tuna/tunasync/tunasync.py -c /etc/tunasync.ini -KillSignal = SIGTERM -ExecReload = /bin/kill -SIGUSR1 $MAINPID -Environment = "HOME=/home/tuna" - -[Install] -WantedBy = multi-user.target diff --git a/tunasync.py b/tunasync.py deleted file mode 100644 index 4f1ce8f..0000000 --- a/tunasync.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import os -import argparse - -from tunasync import TUNASync - - -if __name__ == "__main__": - here = os.path.abspath(os.path.dirname(__file__)) - - parser = argparse.ArgumentParser(prog="tunasync") - parser.add_argument("-c", "--config", - default="tunasync.ini", help="config file") - parser.add_argument("--pidfile", default="/run/tunasync/tunasync.pid", - help="pidfile") - - args = parser.parse_args() - - with open(args.pidfile, 'w') as f: - f.write("{}".format(os.getpid())) - - tunaSync = TUNASync() - tunaSync.read_config(args.config) - - tunaSync.run_jobs() - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync_snapshot_gc.py b/tunasync_snapshot_gc.py deleted file mode 100644 index 0bab1bd..0000000 --- a/tunasync_snapshot_gc.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import re -import sh -import os -import argparse -import toml - -if __name__ == "__main__": - parser = argparse.ArgumentParser(prog="tunasync_snapshot_gc") - parser.add_argument("--max-level", type=int, default=1, help="max walk level to find garbage snapshots") - parser.add_argument("--pattern", default=r"^_gc_.+_\d+", help="pattern to match garbage snapshots") - parser.add_argument("-c", "--config", help="tunasync config file") - - args = parser.parse_args() - - pattern = re.compile(args.pattern) - - def walk(_dir, level=1): - if level > args.max_level: - return - - for fname in os.listdir(_dir): - abs_fname = os.path.join(_dir, fname) - if os.path.isdir(abs_fname): - if pattern.match(fname): - print("GC: {}".format(abs_fname)) - try: - sh.btrfs("subvolume", "delete", abs_fname) - except sh.ErrorReturnCode as e: - print("Error: {}".format(e.stderr)) - else: - walk(abs_fname, level+1) - - with open(args.config) as f: - settings = toml.loads(f.read()) - - mirror_root = settings["global"]["mirror_root"] - gc_root = settings["btrfs"]["gc_root"].format(mirror_root=mirror_root) - - walk(gc_root) - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasynctl.py b/tunasynctl.py deleted file mode 100755 index 14621d8..0000000 --- a/tunasynctl.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import sys -import socket -import argparse -import json -import struct - -if __name__ == "__main__": - parser = argparse.ArgumentParser(prog="tunasynctl") - parser.add_argument("-s", "--socket", - default="/run/tunasync/tunasync.sock", help="socket file") - - subparsers = parser.add_subparsers(dest="command", help='sub-command help') - - sp = subparsers.add_parser('start', help="start job") - sp.add_argument("target", help="mirror job name") - - sp = subparsers.add_parser('stop', help="stop job") - sp.add_argument("target", help="mirror job name") - - sp = subparsers.add_parser('restart', help="restart job") - sp.add_argument("target", help="mirror job name") - - sp = subparsers.add_parser('status', help="show mirror status") - sp.add_argument("target", nargs="?", default="__ALL__", help="mirror job name") - - sp = subparsers.add_parser('log', help="return log file path") - sp.add_argument("-n", type=int, default=0, help="last n-th log, default 0 (latest)") - sp.add_argument("target", help="mirror job name") - - sp = subparsers.add_parser('help', help="show help message") - - args = vars(parser.parse_args()) - - if args['command'] == "help": - parser.print_help() - sys.exit(0) - - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - - try: - sock.connect(args.pop("socket")) - except socket.error as msg: - print(msg) - sys.exit(1) - - pack = json.dumps({ - "cmd": args.pop("command"), - "target": args.pop("target"), - "kwargs": args, - }) - - try: - sock.sendall(struct.pack('!H', len(pack)) + pack) - length = struct.unpack('!H', sock.recv(2))[0] - print(sock.recv(length)) - - except Exception as e: - print(e) - finally: - sock.close() - -# vim: ts=4 sw=4 sts=4 expandtab From ed69dde18e155ad44edc9bb0910186fe4fee3898 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 21 Apr 2016 20:03:07 +0800 Subject: [PATCH 09/66] refactor: moved mirrorStatus back to manager --- internal/status.go | 66 ++++++++++++++++++++ internal/status_test.go | 23 +++++++ {internal/status => manager}/status.go | 68 +++------------------ {internal/status => manager}/status_test.go | 10 +-- 4 files changed, 102 insertions(+), 65 deletions(-) create mode 100644 internal/status.go create mode 100644 internal/status_test.go rename {internal/status => manager}/status.go (64%) rename {internal/status => manager}/status_test.go (86%) diff --git a/internal/status.go b/internal/status.go new file mode 100644 index 0000000..8d20a73 --- /dev/null +++ b/internal/status.go @@ -0,0 +1,66 @@ +package internal + +import ( + "encoding/json" + "errors" + "fmt" +) + +type SyncStatus uint8 + +const ( + None SyncStatus = iota + Failed + Success + Syncing + PreSyncing + Paused + Disabled +) + +func (s SyncStatus) MarshalJSON() ([]byte, error) { + var strStatus string + switch s { + case None: + strStatus = "none" + case Failed: + strStatus = "failed" + case Success: + strStatus = "success" + case Syncing: + strStatus = "syncing" + case PreSyncing: + strStatus = "pre-syncing" + case Paused: + strStatus = "paused" + case Disabled: + strStatus = "disabled" + default: + return []byte{}, errors.New("Invalid status value") + } + + return json.Marshal(strStatus) +} + +func (s *SyncStatus) UnmarshalJSON(v []byte) error { + sv := string(v) + switch sv { + case `"none"`: + *s = None + case `"failed"`: + *s = Failed + case `"success"`: + *s = Success + case `"syncing"`: + *s = Syncing + case `"pre-syncing"`: + *s = PreSyncing + case `"paused"`: + *s = Paused + case `"disabled"`: + *s = Disabled + default: + return fmt.Errorf("Invalid status value: %s", string(v)) + } + return nil +} diff --git a/internal/status_test.go b/internal/status_test.go new file mode 100644 index 0000000..fcfafe8 --- /dev/null +++ b/internal/status_test.go @@ -0,0 +1,23 @@ +package internal + +import ( + "encoding/json" + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestSyncStatus(t *testing.T) { + Convey("SyncStatus json ser-de should work", t, func() { + + b, err := json.Marshal(PreSyncing) + So(err, ShouldBeNil) + So(b, ShouldResemble, []byte(`"pre-syncing"`)) // deep equal should be used + + var s SyncStatus + + err = json.Unmarshal([]byte(`"failed"`), &s) + So(err, ShouldBeNil) + So(s, ShouldEqual, Failed) + }) +} diff --git a/internal/status/status.go b/manager/status.go similarity index 64% rename from internal/status/status.go rename to manager/status.go index dfab49c..c9e2c90 100644 --- a/internal/status/status.go +++ b/manager/status.go @@ -1,35 +1,24 @@ -package status +package manager import ( "encoding/json" "errors" "fmt" "strconv" - "strings" "time" + + . "github.com/tuna/tunasync/internal" ) -type syncStatus uint8 - -const ( - None syncStatus = iota - Failed - Success - Syncing - PreSyncing - Paused - Disabled -) - -type MirrorStatus struct { +type mirrorStatus struct { Name string - Status syncStatus + Status SyncStatus LastUpdate time.Time Upstream string Size string // approximate size } -func (s MirrorStatus) MarshalJSON() ([]byte, error) { +func (s mirrorStatus) MarshalJSON() ([]byte, error) { m := map[string]interface{}{ "name": s.Name, "status": s.Status, @@ -41,7 +30,7 @@ func (s MirrorStatus) MarshalJSON() ([]byte, error) { return json.Marshal(m) } -func (s *MirrorStatus) UnmarshalJSON(v []byte) error { +func (s *mirrorStatus) UnmarshalJSON(v []byte) error { var m map[string]interface{} err := json.Unmarshal(v, &m) @@ -99,46 +88,3 @@ func (s *MirrorStatus) UnmarshalJSON(v []byte) error { } return nil } - -func (s syncStatus) MarshalJSON() ([]byte, error) { - var strStatus string - switch s { - case None: - strStatus = "none" - case Success: - strStatus = "success" - case Syncing: - strStatus = "syncing" - case PreSyncing: - strStatus = "pre-syncing" - case Paused: - strStatus = "paused" - case Disabled: - strStatus = "disabled" - default: - return []byte{}, errors.New("Invalid status value") - } - - return json.Marshal(strStatus) -} - -func (s *syncStatus) UnmarshalJSON(v []byte) error { - sv := strings.Trim(string(v), `"`) - switch sv { - case "none": - *s = None - case "success": - *s = Success - case "syncing": - *s = Syncing - case "pre-syncing": - *s = PreSyncing - case "paused": - *s = Paused - case "disabled": - *s = Disabled - default: - return fmt.Errorf("Invalid status value: %s", string(v)) - } - return nil -} diff --git a/internal/status/status_test.go b/manager/status_test.go similarity index 86% rename from internal/status/status_test.go rename to manager/status_test.go index 578a115..06260d9 100644 --- a/internal/status/status_test.go +++ b/manager/status_test.go @@ -1,10 +1,12 @@ -package status +package manager import ( "encoding/json" "testing" "time" + tunasync "github.com/tuna/tunasync/internal" + . "github.com/smartystreets/goconvey/convey" ) @@ -14,9 +16,9 @@ func TestStatus(t *testing.T) { loc, err := time.LoadLocation(tz) So(err, ShouldBeNil) - m := MirrorStatus{ + m := mirrorStatus{ Name: "tunalinux", - Status: Success, + Status: tunasync.Success, LastUpdate: time.Date(2016, time.April, 16, 23, 8, 10, 0, loc), Size: "5GB", Upstream: "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/", @@ -25,7 +27,7 @@ func TestStatus(t *testing.T) { b, err := json.Marshal(m) So(err, ShouldBeNil) // fmt.Println(string(b)) - var m2 MirrorStatus + var m2 mirrorStatus err = json.Unmarshal(b, &m2) So(err, ShouldBeNil) // fmt.Printf("%#v", m2) From f0a0552e50367c75a04c0973267b58dbc9641e25 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 21 Apr 2016 20:07:28 +0800 Subject: [PATCH 10/66] chore: modified test files --- .gitignore | 1 + .testpackages.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7bdd5d8..e96a241 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,4 @@ target/ *.swp *~ /examples/tunasync.json +/*.cov diff --git a/.testpackages.txt b/.testpackages.txt index 9e724cc..ba04577 100644 --- a/.testpackages.txt +++ b/.testpackages.txt @@ -1,3 +1,2 @@ github.com/tuna/tunasync/internal -github.com/tuna/tunasync/internal/status github.com/tuna/tunasync/manager From f95a0f9a6fd0dd0ee5d6f450edf3b3e83ab1066b Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 21 Apr 2016 20:45:06 +0800 Subject: [PATCH 11/66] feature(worker): context object to store runtime configurations --- worker/context.go | 61 ++++++++++++++++++++++++++++++++++++++++ worker/context_test.go | 64 ++++++++++++++++++++++++++++++++++++++++++ worker/provider.go | 13 +++++++++ 3 files changed, 138 insertions(+) create mode 100644 worker/context.go create mode 100644 worker/context_test.go create mode 100644 worker/provider.go diff --git a/worker/context.go b/worker/context.go new file mode 100644 index 0000000..7a240a7 --- /dev/null +++ b/worker/context.go @@ -0,0 +1,61 @@ +package worker + +// Context object aims to store runtime configurations + +import "errors" + +// A Context object is a layered key-value storage +// when enters a context, the changes to the storage would be stored +// in a new layer and when exits, the top layer poped and the storage +// returned to the state before entering this context +type Context struct { + parent *Context + store map[string]interface{} +} + +// NewContext returns a new context object +func NewContext() *Context { + return &Context{ + parent: nil, + store: make(map[string]interface{}), + } +} + +// Enter generates a new layer of context +func (ctx *Context) Enter() *Context { + + return &Context{ + parent: ctx, + store: make(map[string]interface{}), + } + +} + +// Exit return the upper layer of context +func (ctx *Context) Exit() (*Context, error) { + if ctx.parent == nil { + return nil, errors.New("Cannot exit the bottom layer context") + } + return ctx.parent, nil +} + +// Get returns the value corresponding to key, if it's +// not found in the current layer, return the lower layer +// context's value +func (ctx *Context) Get(key string) (interface{}, bool) { + if ctx.parent == nil { + if value, ok := ctx.store[key]; ok { + return value, true + } + return nil, false + } + if value, ok := ctx.store[key]; ok { + return value, true + } + return ctx.parent.Get(key) +} + +// Set sets the value to the key at current layer +func (ctx *Context) Set(key string, value interface{}) { + ctx.store[key] = value +} diff --git a/worker/context_test.go b/worker/context_test.go new file mode 100644 index 0000000..f11c0ab --- /dev/null +++ b/worker/context_test.go @@ -0,0 +1,64 @@ +package worker + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestContext(t *testing.T) { + Convey("Context should work", t, func() { + + ctx := NewContext() + So(ctx, ShouldNotBeNil) + So(ctx.parent, ShouldBeNil) + + ctx.Set("logdir1", "logdir_value_1") + ctx.Set("logdir2", "logdir_value_2") + logdir, ok := ctx.Get("logdir1") + So(ok, ShouldBeTrue) + So(logdir, ShouldEqual, "logdir_value_1") + + Convey("When entering a new context", func() { + ctx = ctx.Enter() + logdir, ok = ctx.Get("logdir1") + So(ok, ShouldBeTrue) + So(logdir, ShouldEqual, "logdir_value_1") + + ctx.Set("logdir1", "new_value_1") + + logdir, ok = ctx.Get("logdir1") + So(ok, ShouldBeTrue) + So(logdir, ShouldEqual, "new_value_1") + + logdir, ok = ctx.Get("logdir2") + So(ok, ShouldBeTrue) + So(logdir, ShouldEqual, "logdir_value_2") + + Convey("When accesing invalid key", func() { + logdir, ok = ctx.Get("invalid_key") + So(ok, ShouldBeFalse) + So(logdir, ShouldBeNil) + }) + + Convey("When exiting the new context", func() { + ctx, err := ctx.Exit() + So(err, ShouldBeNil) + + logdir, ok = ctx.Get("logdir1") + So(ok, ShouldBeTrue) + So(logdir, ShouldEqual, "logdir_value_1") + + logdir, ok = ctx.Get("logdir2") + So(ok, ShouldBeTrue) + So(logdir, ShouldEqual, "logdir_value_2") + + Convey("When exiting from top bottom context", func() { + ctx, err := ctx.Exit() + So(err, ShouldNotBeNil) + So(ctx, ShouldBeNil) + }) + }) + }) + }) +} diff --git a/worker/provider.go b/worker/provider.go new file mode 100644 index 0000000..410e93a --- /dev/null +++ b/worker/provider.go @@ -0,0 +1,13 @@ +// mirror provider is the wrapper of mirror jobs + +package worker + +// a mirrorProvider instance +type mirrorProvider interface { + // run mirror job + Run() + // terminate mirror job + Terminate() + // get context + Context() +} From 44af0d5e62faf7846b8d0c2a8dbc12970a3bc9c2 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 21 Apr 2016 21:57:32 +0800 Subject: [PATCH 12/66] feature(worker): framework of mirror provider --- .testpackages.txt | 1 + worker/provider.go | 90 +++++++++++++++++++++++++++++++++++++--- worker/provider_test.go | 59 ++++++++++++++++++++++++++ worker/rsync_provider.go | 48 +++++++++++++++++++++ 4 files changed, 193 insertions(+), 5 deletions(-) create mode 100644 worker/provider_test.go create mode 100644 worker/rsync_provider.go diff --git a/.testpackages.txt b/.testpackages.txt index ba04577..f95aed0 100644 --- a/.testpackages.txt +++ b/.testpackages.txt @@ -1,2 +1,3 @@ github.com/tuna/tunasync/internal github.com/tuna/tunasync/manager +github.com/tuna/tunasync/worker diff --git a/worker/provider.go b/worker/provider.go index 410e93a..065c40e 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -1,13 +1,93 @@ -// mirror provider is the wrapper of mirror jobs - package worker -// a mirrorProvider instance +// mirror provider is the wrapper of mirror jobs + +type providerType uint8 + +const ( + _WorkingDirKey = "working_dir" + _LogDirKey = "log_dir" + _LogFileKey = "log_file" +) + +// A mirrorProvider instance type mirrorProvider interface { + // name + Name() string + + // TODO: implement Run, Terminate and Hooks // run mirror job Run() // terminate mirror job Terminate() - // get context - Context() + // job hooks + Hooks() + + Interval() int + + WorkingDir() string + LogDir() string + LogFile() string + + // enter context + EnterContext() *Context + // exit context + ExitContext() *Context + // return context + Context() *Context +} + +type baseProvider struct { + ctx *Context + name string + interval int +} + +func (p *baseProvider) Name() string { + return p.name +} + +func (p *baseProvider) EnterContext() *Context { + p.ctx = p.ctx.Enter() + return p.ctx +} + +func (p *baseProvider) ExitContext() *Context { + p.ctx, _ = p.ctx.Exit() + return p.ctx +} + +func (p *baseProvider) Context() *Context { + return p.ctx +} + +func (p *baseProvider) Interval() int { + return p.interval +} + +func (p *baseProvider) WorkingDir() string { + if v, ok := p.ctx.Get(_WorkingDirKey); ok { + if s, ok := v.(string); ok { + return s + } + } + panic("working dir is impossible to be non-exist") +} + +func (p *baseProvider) LogDir() string { + if v, ok := p.ctx.Get(_LogDirKey); ok { + if s, ok := v.(string); ok { + return s + } + } + panic("log dir is impossible to be unavailable") +} + +func (p *baseProvider) LogFile() string { + if v, ok := p.ctx.Get(_LogFileKey); ok { + if s, ok := v.(string); ok { + return s + } + } + panic("log dir is impossible to be unavailable") } diff --git a/worker/provider_test.go b/worker/provider_test.go new file mode 100644 index 0000000..7374da8 --- /dev/null +++ b/worker/provider_test.go @@ -0,0 +1,59 @@ +package worker + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestRsyncProvider(t *testing.T) { + Convey("Rsync Provider should work", t, func() { + + c := rsyncConfig{ + name: "tuna", + upstreamURL: "rsync://rsync.tuna.moe/tuna/", + workingDir: "/srv/mirror/production/tuna", + logDir: "/var/log/tunasync", + logFile: "tuna.log", + useIPv6: true, + interval: 600, + } + + provider, err := newRsyncProvider(c) + So(err, ShouldBeNil) + + So(provider.Name(), ShouldEqual, c.name) + So(provider.WorkingDir(), ShouldEqual, c.workingDir) + So(provider.LogDir(), ShouldEqual, c.logDir) + So(provider.LogFile(), ShouldEqual, c.logFile) + + Convey("When entering a context (auto exit)", func() { + func() { + ctx := provider.EnterContext() + defer provider.ExitContext() + So(provider.WorkingDir(), ShouldEqual, c.workingDir) + newWorkingDir := "/srv/mirror/working/tuna" + ctx.Set(_WorkingDirKey, newWorkingDir) + So(provider.WorkingDir(), ShouldEqual, newWorkingDir) + }() + + Convey("After context is done", func() { + So(provider.WorkingDir(), ShouldEqual, c.workingDir) + }) + }) + + Convey("When entering a context (manually exit)", func() { + ctx := provider.EnterContext() + So(provider.WorkingDir(), ShouldEqual, c.workingDir) + newWorkingDir := "/srv/mirror/working/tuna" + ctx.Set(_WorkingDirKey, newWorkingDir) + So(provider.WorkingDir(), ShouldEqual, newWorkingDir) + + Convey("After context is done", func() { + provider.ExitContext() + So(provider.WorkingDir(), ShouldEqual, c.workingDir) + }) + }) + + }) +} diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go new file mode 100644 index 0000000..b0bcd06 --- /dev/null +++ b/worker/rsync_provider.go @@ -0,0 +1,48 @@ +package worker + +type rsyncConfig struct { + name string + upstreamURL, password, excludeFile string + workingDir, logDir, logFile string + useIPv6 bool + interval int +} + +// An RsyncProvider provides the implementation to rsync-based syncing jobs +type rsyncProvider struct { + baseProvider + rsyncConfig +} + +func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { + // TODO: check config options + provider := &rsyncProvider{ + baseProvider: baseProvider{ + name: c.name, + ctx: NewContext(), + interval: c.interval, + }, + rsyncConfig: c, + } + + provider.ctx.Set(_WorkingDirKey, c.workingDir) + provider.ctx.Set(_LogDirKey, c.logDir) + provider.ctx.Set(_LogFileKey, c.logFile) + + return provider, nil +} + +// TODO: implement this +func (p *rsyncProvider) Run() { + +} + +// TODO: implement this +func (p *rsyncProvider) Terminate() { + +} + +// TODO: implement this +func (p *rsyncProvider) Hooks() { + +} From 6948db175714c25ee2f2dc8825d6f0d2011b7b46 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 21 Apr 2016 23:35:49 +0800 Subject: [PATCH 13/66] feature(worker): cmd_provider, first part, no test, fix me --- worker/cmd_provider.go | 114 ++++++++++++++++++++++++++++++++++++++++ worker/provider_test.go | 1 + 2 files changed, 115 insertions(+) create mode 100644 worker/cmd_provider.go diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go new file mode 100644 index 0000000..36cc700 --- /dev/null +++ b/worker/cmd_provider.go @@ -0,0 +1,114 @@ +package worker + +import ( + "os" + "os/exec" + "strings" + + "github.com/anmitsu/go-shlex" + "github.com/codeskyblue/go-sh" +) + +type cmdConfig struct { + name string + upstreamURL, command string + workingDir, logDir, logFile string + interval int + env map[string]string +} + +type cmdProvider struct { + baseProvider + cmdConfig + cmd []string + session *sh.Session +} + +func newCmdProvider(c cmdConfig) (*cmdProvider, error) { + // TODO: check config options + provider := &cmdProvider{ + baseProvider: baseProvider{ + name: c.name, + ctx: NewContext(), + interval: c.interval, + }, + cmdConfig: c, + } + + provider.ctx.Set(_WorkingDirKey, c.workingDir) + provider.ctx.Set(_LogDirKey, c.logDir) + provider.ctx.Set(_LogFileKey, c.logFile) + + cmd, err := shlex.Split(c.command, true) + if err != nil { + return nil, err + } + provider.cmd = cmd + + return provider, nil +} + +// Copied from go-sh +func newEnviron(env map[string]string, inherit bool) []string { //map[string]string { + environ := make([]string, 0, len(env)) + if inherit { + for _, line := range os.Environ() { + // if os environment and env collapses, + // omit the os one + k := strings.Split(line, "=")[0] + if _, ok := env[k]; ok { + continue + } + environ = append(environ, line) + } + } + for k, v := range env { + environ = append(environ, k+"="+v) + } + return environ +} + +// TODO: implement this +func (p *cmdProvider) Run() error { + var cmd *exec.Cmd + if len(p.cmd) == 1 { + cmd = exec.Command(p.cmd[0]) + } else if len(p.cmd) > 1 { + c := p.cmd[0] + args := p.cmd[1:] + cmd = exec.Command(c, args...) + } else if len(p.cmd) == 0 { + panic("Command length should be at least 1!") + } + cmd.Dir = p.WorkingDir() + + env := map[string]string{ + "TUNASYNC_MIRROR_NAME": p.Name(), + "TUNASYNC_WORKING_DIR": p.WorkingDir(), + "TUNASYNC_UPSTREAM_URL": p.upstreamURL, + "TUNASYNC_LOG_FILE": p.LogFile(), + } + for k, v := range p.env { + env[k] = v + } + cmd.Env = newEnviron(env, true) + + logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY, 0644) + if err != nil { + return err + } + cmd.Stdout = logFile + cmd.Stderr = logFile + + return cmd.Start() +} + +// TODO: implement this +func (p *cmdProvider) Terminate() { + +} + +// TODO: implement this +func (p *cmdProvider) Hooks() { + +} diff --git a/worker/provider_test.go b/worker/provider_test.go index 7374da8..ce09316 100644 --- a/worker/provider_test.go +++ b/worker/provider_test.go @@ -26,6 +26,7 @@ func TestRsyncProvider(t *testing.T) { So(provider.WorkingDir(), ShouldEqual, c.workingDir) So(provider.LogDir(), ShouldEqual, c.logDir) So(provider.LogFile(), ShouldEqual, c.logFile) + So(provider.Interval(), ShouldEqual, c.interval) Convey("When entering a context (auto exit)", func() { func() { From 16ead051607ca514c9377ad57a9e44307855a280 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 22 Apr 2016 09:19:11 +0800 Subject: [PATCH 14/66] tests(worker): command provider's test --- worker/cmd_provider.go | 36 ++++++++++--------- worker/provider_test.go | 78 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 16 deletions(-) diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go index 36cc700..f6ee4aa 100644 --- a/worker/cmd_provider.go +++ b/worker/cmd_provider.go @@ -20,7 +20,8 @@ type cmdConfig struct { type cmdProvider struct { baseProvider cmdConfig - cmd []string + command []string + cmd *exec.Cmd session *sh.Session } @@ -43,7 +44,7 @@ func newCmdProvider(c cmdConfig) (*cmdProvider, error) { if err != nil { return nil, err } - provider.cmd = cmd + provider.command = cmd return provider, nil } @@ -70,17 +71,16 @@ func newEnviron(env map[string]string, inherit bool) []string { //map[string]str // TODO: implement this func (p *cmdProvider) Run() error { - var cmd *exec.Cmd - if len(p.cmd) == 1 { - cmd = exec.Command(p.cmd[0]) - } else if len(p.cmd) > 1 { - c := p.cmd[0] - args := p.cmd[1:] - cmd = exec.Command(c, args...) - } else if len(p.cmd) == 0 { + if len(p.command) == 1 { + p.cmd = exec.Command(p.command[0]) + } else if len(p.command) > 1 { + c := p.command[0] + args := p.command[1:] + p.cmd = exec.Command(c, args...) + } else if len(p.command) == 0 { panic("Command length should be at least 1!") } - cmd.Dir = p.WorkingDir() + p.cmd.Dir = p.WorkingDir() env := map[string]string{ "TUNASYNC_MIRROR_NAME": p.Name(), @@ -91,16 +91,20 @@ func (p *cmdProvider) Run() error { for k, v := range p.env { env[k] = v } - cmd.Env = newEnviron(env, true) + p.cmd.Env = newEnviron(env, true) - logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY, 0644) + logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY|os.O_CREATE, 0644) if err != nil { return err } - cmd.Stdout = logFile - cmd.Stderr = logFile + p.cmd.Stdout = logFile + p.cmd.Stderr = logFile - return cmd.Start() + return p.cmd.Start() +} + +func (p *cmdProvider) Wait() error { + return p.cmd.Wait() } // TODO: implement this diff --git a/worker/provider_test.go b/worker/provider_test.go index ce09316..aa2ada2 100644 --- a/worker/provider_test.go +++ b/worker/provider_test.go @@ -1,6 +1,10 @@ package worker import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" "testing" . "github.com/smartystreets/goconvey/convey" @@ -58,3 +62,77 @@ func TestRsyncProvider(t *testing.T) { }) } + +func TestCmdProvider(t *testing.T) { + Convey("Command Provider should work", t, func() { + tmpDir, err := ioutil.TempDir("", "tunasync") + defer os.RemoveAll(tmpDir) + So(err, ShouldBeNil) + scriptFile := filepath.Join(tmpDir, "cmd.sh") + tmpFile := filepath.Join(tmpDir, "log_file") + + c := cmdConfig{ + name: "tuna-cmd", + upstreamURL: "http://mirrors.tuna.moe/", + command: "bash " + scriptFile, + workingDir: tmpDir, + logDir: tmpDir, + logFile: tmpFile, + interval: 600, + } + + provider, err := newCmdProvider(c) + So(err, ShouldBeNil) + + So(provider.Name(), ShouldEqual, c.name) + So(provider.WorkingDir(), ShouldEqual, c.workingDir) + So(provider.LogDir(), ShouldEqual, c.logDir) + So(provider.LogFile(), ShouldEqual, c.logFile) + So(provider.Interval(), ShouldEqual, c.interval) + + Convey("Let's try to run a simple command", func() { + scriptContent := `#!/bin/bash +echo $TUNASYNC_WORKING_DIR +echo $TUNASYNC_MIRROR_NAME +echo $TUNASYNC_UPSTREAM_URL +echo $TUNASYNC_LOG_FILE +` + exceptedOutput := fmt.Sprintf( + "%s\n%s\n%s\n%s\n", + provider.WorkingDir(), + provider.Name(), + provider.upstreamURL, + provider.LogFile(), + ) + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + readedScriptContent, err := ioutil.ReadFile(scriptFile) + So(err, ShouldBeNil) + So(readedScriptContent, ShouldResemble, []byte(scriptContent)) + + err = provider.Run() + So(err, ShouldBeNil) + err = provider.cmd.Wait() + So(err, ShouldBeNil) + + loggedContent, err := ioutil.ReadFile(provider.LogFile()) + So(err, ShouldBeNil) + So(string(loggedContent), ShouldEqual, exceptedOutput) + }) + + Convey("If a command fails", func() { + scriptContent := `exit 1` + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + readedScriptContent, err := ioutil.ReadFile(scriptFile) + So(err, ShouldBeNil) + So(readedScriptContent, ShouldResemble, []byte(scriptContent)) + + err = provider.Run() + So(err, ShouldBeNil) + err = provider.cmd.Wait() + So(err, ShouldNotBeNil) + + }) + }) +} From 276ab233c5446544c17d222d4a82002b79c2b24e Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 22 Apr 2016 11:18:52 +0800 Subject: [PATCH 15/66] feature(worker): move command execution logic to a runner object --- worker/cmd_provider.go | 53 +++++-------------- worker/provider_test.go | 28 ++++++++-- worker/runner.go | 112 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 44 deletions(-) create mode 100644 worker/runner.go diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go index f6ee4aa..ca808ad 100644 --- a/worker/cmd_provider.go +++ b/worker/cmd_provider.go @@ -1,12 +1,10 @@ package worker import ( + "errors" "os" - "os/exec" - "strings" "github.com/anmitsu/go-shlex" - "github.com/codeskyblue/go-sh" ) type cmdConfig struct { @@ -21,8 +19,7 @@ type cmdProvider struct { baseProvider cmdConfig command []string - cmd *exec.Cmd - session *sh.Session + cmd *cmdJob } func newCmdProvider(c cmdConfig) (*cmdProvider, error) { @@ -49,39 +46,10 @@ func newCmdProvider(c cmdConfig) (*cmdProvider, error) { return provider, nil } -// Copied from go-sh -func newEnviron(env map[string]string, inherit bool) []string { //map[string]string { - environ := make([]string, 0, len(env)) - if inherit { - for _, line := range os.Environ() { - // if os environment and env collapses, - // omit the os one - k := strings.Split(line, "=")[0] - if _, ok := env[k]; ok { - continue - } - environ = append(environ, line) - } - } - for k, v := range env { - environ = append(environ, k+"="+v) - } - return environ +func (p *cmdProvider) InitRunner() { } -// TODO: implement this func (p *cmdProvider) Run() error { - if len(p.command) == 1 { - p.cmd = exec.Command(p.command[0]) - } else if len(p.command) > 1 { - c := p.command[0] - args := p.command[1:] - p.cmd = exec.Command(c, args...) - } else if len(p.command) == 0 { - panic("Command length should be at least 1!") - } - p.cmd.Dir = p.WorkingDir() - env := map[string]string{ "TUNASYNC_MIRROR_NAME": p.Name(), "TUNASYNC_WORKING_DIR": p.WorkingDir(), @@ -91,14 +59,14 @@ func (p *cmdProvider) Run() error { for k, v := range p.env { env[k] = v } - p.cmd.Env = newEnviron(env, true) + p.cmd = newCmdJob(p.command, p.WorkingDir(), env) logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY|os.O_CREATE, 0644) if err != nil { return err } - p.cmd.Stdout = logFile - p.cmd.Stderr = logFile + // defer logFile.Close() + p.cmd.SetLogFile(logFile) return p.cmd.Start() } @@ -107,9 +75,12 @@ func (p *cmdProvider) Wait() error { return p.cmd.Wait() } -// TODO: implement this -func (p *cmdProvider) Terminate() { - +func (p *cmdProvider) Terminate() error { + if p.cmd == nil { + return errors.New("provider command job not initialized") + } + err := p.cmd.Terminate() + return err } // TODO: implement this diff --git a/worker/provider_test.go b/worker/provider_test.go index aa2ada2..f249612 100644 --- a/worker/provider_test.go +++ b/worker/provider_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "testing" + "time" . "github.com/smartystreets/goconvey/convey" ) @@ -64,7 +65,7 @@ func TestRsyncProvider(t *testing.T) { } func TestCmdProvider(t *testing.T) { - Convey("Command Provider should work", t, func() { + Convey("Command Provider should work", t, func(ctx C) { tmpDir, err := ioutil.TempDir("", "tunasync") defer os.RemoveAll(tmpDir) So(err, ShouldBeNil) @@ -112,7 +113,7 @@ echo $TUNASYNC_LOG_FILE err = provider.Run() So(err, ShouldBeNil) - err = provider.cmd.Wait() + err = provider.Wait() So(err, ShouldBeNil) loggedContent, err := ioutil.ReadFile(provider.LogFile()) @@ -130,9 +131,30 @@ echo $TUNASYNC_LOG_FILE err = provider.Run() So(err, ShouldBeNil) - err = provider.cmd.Wait() + err = provider.Wait() So(err, ShouldNotBeNil) }) + + Convey("If a long job is killed", func(ctx C) { + scriptContent := `#!/bin/bash +sleep 5 + ` + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + + err = provider.Run() + So(err, ShouldBeNil) + + go func() { + err = provider.Wait() + ctx.So(err, ShouldNotBeNil) + }() + + time.Sleep(2) + err = provider.Terminate() + So(err, ShouldBeNil) + + }) }) } diff --git a/worker/runner.go b/worker/runner.go new file mode 100644 index 0000000..9693ddb --- /dev/null +++ b/worker/runner.go @@ -0,0 +1,112 @@ +package worker + +import ( + "errors" + "os" + "os/exec" + "strings" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +// runner is to run os commands giving command line, env and log file +// it's an alternative to python-sh or go-sh +// TODO: cgroup excution + +type cmdJob struct { + cmd *exec.Cmd + workingDir string + env map[string]string + logFile *os.File + finished chan struct{} +} + +func newCmdJob(cmdAndArgs []string, workingDir string, env map[string]string) *cmdJob { + var cmd *exec.Cmd + if len(cmdAndArgs) == 1 { + cmd = exec.Command(cmdAndArgs[0]) + } else if len(cmdAndArgs) > 1 { + c := cmdAndArgs[0] + args := cmdAndArgs[1:] + cmd = exec.Command(c, args...) + } else if len(cmdAndArgs) == 0 { + panic("Command length should be at least 1!") + } + + cmd.Dir = workingDir + cmd.Env = newEnviron(env, true) + + return &cmdJob{ + cmd: cmd, + workingDir: workingDir, + env: env, + } +} + +// start job and wait +func (c *cmdJob) Run() error { + err := c.cmd.Start() + if err != nil { + return err + } + return c.Wait() +} + +func (c *cmdJob) Start() error { + c.finished = make(chan struct{}, 1) + return c.cmd.Start() +} + +func (c *cmdJob) Wait() error { + err := c.cmd.Wait() + c.finished <- struct{}{} + return err +} + +func (c *cmdJob) SetLogFile(logFile *os.File) { + c.cmd.Stdout = logFile + c.cmd.Stderr = logFile +} + +func (c *cmdJob) Terminate() error { + if c.cmd == nil { + return errors.New("Command not initialized") + } + if c.cmd.Process == nil { + return errors.New("No Process Running") + } + err := unix.Kill(c.cmd.Process.Pid, syscall.SIGTERM) + if err != nil { + return err + } + + select { + case <-time.After(2 * time.Second): + unix.Kill(c.cmd.Process.Pid, syscall.SIGKILL) + return errors.New("SIGTERM failed to kill the job") + case <-c.finished: + return nil + } +} + +// Copied from go-sh +func newEnviron(env map[string]string, inherit bool) []string { //map[string]string { + environ := make([]string, 0, len(env)) + if inherit { + for _, line := range os.Environ() { + // if os environment and env collapses, + // omit the os one + k := strings.Split(line, "=")[0] + if _, ok := env[k]; ok { + continue + } + environ = append(environ, line) + } + } + for k, v := range env { + environ = append(environ, k+"="+v) + } + return environ +} From 0e808a449a8ed7658dfea962eed3059e6e32ec90 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 22 Apr 2016 20:59:43 +0800 Subject: [PATCH 16/66] refactor(worker): change provider's Run method to Start, and change logfile handling --- worker/cmd_provider.go | 14 +++++++++----- worker/provider.go | 6 ++++-- worker/provider_test.go | 13 +++++++++---- worker/rsync_provider.go | 2 +- worker/runner.go | 9 --------- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go index ca808ad..b97933f 100644 --- a/worker/cmd_provider.go +++ b/worker/cmd_provider.go @@ -20,6 +20,7 @@ type cmdProvider struct { cmdConfig command []string cmd *cmdJob + logFile *os.File } func newCmdProvider(c cmdConfig) (*cmdProvider, error) { @@ -46,10 +47,7 @@ func newCmdProvider(c cmdConfig) (*cmdProvider, error) { return provider, nil } -func (p *cmdProvider) InitRunner() { -} - -func (p *cmdProvider) Run() error { +func (p *cmdProvider) Start() error { env := map[string]string{ "TUNASYNC_MIRROR_NAME": p.Name(), "TUNASYNC_WORKING_DIR": p.WorkingDir(), @@ -65,13 +63,16 @@ func (p *cmdProvider) Run() error { if err != nil { return err } - // defer logFile.Close() + p.logFile = logFile p.cmd.SetLogFile(logFile) return p.cmd.Start() } func (p *cmdProvider) Wait() error { + if p.logFile != nil { + defer p.logFile.Close() + } return p.cmd.Wait() } @@ -79,6 +80,9 @@ func (p *cmdProvider) Terminate() error { if p.cmd == nil { return errors.New("provider command job not initialized") } + if p.logFile != nil { + defer p.logFile.Close() + } err := p.cmd.Terminate() return err } diff --git a/worker/provider.go b/worker/provider.go index 065c40e..806c073 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -16,8 +16,10 @@ type mirrorProvider interface { Name() string // TODO: implement Run, Terminate and Hooks - // run mirror job - Run() + // run mirror job in background + Start() + // Wait job to finish + Wait() // terminate mirror job Terminate() // job hooks diff --git a/worker/provider_test.go b/worker/provider_test.go index f249612..79b5b03 100644 --- a/worker/provider_test.go +++ b/worker/provider_test.go @@ -80,6 +80,9 @@ func TestCmdProvider(t *testing.T) { logDir: tmpDir, logFile: tmpFile, interval: 600, + env: map[string]string{ + "AOSP_REPO_BIN": "/usr/local/bin/repo", + }, } provider, err := newCmdProvider(c) @@ -97,13 +100,15 @@ echo $TUNASYNC_WORKING_DIR echo $TUNASYNC_MIRROR_NAME echo $TUNASYNC_UPSTREAM_URL echo $TUNASYNC_LOG_FILE +echo $AOSP_REPO_BIN ` exceptedOutput := fmt.Sprintf( - "%s\n%s\n%s\n%s\n", + "%s\n%s\n%s\n%s\n%s\n", provider.WorkingDir(), provider.Name(), provider.upstreamURL, provider.LogFile(), + "/usr/local/bin/repo", ) err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) So(err, ShouldBeNil) @@ -111,7 +116,7 @@ echo $TUNASYNC_LOG_FILE So(err, ShouldBeNil) So(readedScriptContent, ShouldResemble, []byte(scriptContent)) - err = provider.Run() + err = provider.Start() So(err, ShouldBeNil) err = provider.Wait() So(err, ShouldBeNil) @@ -129,7 +134,7 @@ echo $TUNASYNC_LOG_FILE So(err, ShouldBeNil) So(readedScriptContent, ShouldResemble, []byte(scriptContent)) - err = provider.Run() + err = provider.Start() So(err, ShouldBeNil) err = provider.Wait() So(err, ShouldNotBeNil) @@ -143,7 +148,7 @@ sleep 5 err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) So(err, ShouldBeNil) - err = provider.Run() + err = provider.Start() So(err, ShouldBeNil) go func() { diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go index b0bcd06..1687109 100644 --- a/worker/rsync_provider.go +++ b/worker/rsync_provider.go @@ -33,7 +33,7 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { } // TODO: implement this -func (p *rsyncProvider) Run() { +func (p *rsyncProvider) Start() { } diff --git a/worker/runner.go b/worker/runner.go index 9693ddb..92edbba 100644 --- a/worker/runner.go +++ b/worker/runner.go @@ -45,15 +45,6 @@ func newCmdJob(cmdAndArgs []string, workingDir string, env map[string]string) *c } } -// start job and wait -func (c *cmdJob) Run() error { - err := c.cmd.Start() - if err != nil { - return err - } - return c.Wait() -} - func (c *cmdJob) Start() error { c.finished = make(chan struct{}, 1) return c.cmd.Start() From 681388ffdd3d290c7a6048a28c855f665fdbf87b Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sat, 23 Apr 2016 17:52:30 +0800 Subject: [PATCH 17/66] feature(worker): toplevel mirror job logic --- worker/cmd_provider.go | 11 +-- worker/common.go | 13 +++ worker/hooks.go | 42 ++++++++ worker/job.go | 202 +++++++++++++++++++++++++++++++++++++++ worker/job_test.go | 135 ++++++++++++++++++++++++++ worker/provider.go | 25 +++-- worker/rsync_provider.go | 9 +- worker/runner.go | 10 +- 8 files changed, 422 insertions(+), 25 deletions(-) create mode 100644 worker/common.go create mode 100644 worker/hooks.go create mode 100644 worker/job.go create mode 100644 worker/job_test.go diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go index b97933f..9bdaa09 100644 --- a/worker/cmd_provider.go +++ b/worker/cmd_provider.go @@ -3,6 +3,7 @@ package worker import ( "errors" "os" + "time" "github.com/anmitsu/go-shlex" ) @@ -11,7 +12,7 @@ type cmdConfig struct { name string upstreamURL, command string workingDir, logDir, logFile string - interval int + interval time.Duration env map[string]string } @@ -77,17 +78,13 @@ func (p *cmdProvider) Wait() error { } func (p *cmdProvider) Terminate() error { + logger.Debug("terminating provider: %s", p.Name()) if p.cmd == nil { return errors.New("provider command job not initialized") } if p.logFile != nil { - defer p.logFile.Close() + p.logFile.Close() } err := p.cmd.Terminate() return err } - -// TODO: implement this -func (p *cmdProvider) Hooks() { - -} diff --git a/worker/common.go b/worker/common.go new file mode 100644 index 0000000..65bdb47 --- /dev/null +++ b/worker/common.go @@ -0,0 +1,13 @@ +package worker + +// put global viables and types here + +import ( + "gopkg.in/op/go-logging.v1" +) + +type empty struct{} + +const maxRetry = 2 + +var logger = logging.MustGetLogger("tunasync") diff --git a/worker/hooks.go b/worker/hooks.go new file mode 100644 index 0000000..b5d318b --- /dev/null +++ b/worker/hooks.go @@ -0,0 +1,42 @@ +package worker + +/* +hooks to exec before/after syncing + failed + +------------------ post-fail hooks -------------------+ + | | + job start -> pre-job hooks --v-> pre-exec hooks --> (syncing) --> post-exec hooks --+---------> post-success --> end + success +*/ + +type jobHook interface { + preJob() error + preExec() error + postExec() error + postSuccess() error + postFail() error +} + +type emptyHook struct { + provider mirrorProvider +} + +func (h *emptyHook) preJob() error { + return nil +} + +func (h *emptyHook) preExec() error { + return nil +} + +func (h *emptyHook) postExec() error { + return nil +} + +func (h *emptyHook) postSuccess() error { + return nil +} + +func (h *emptyHook) postFail() error { + return nil +} diff --git a/worker/job.go b/worker/job.go new file mode 100644 index 0000000..5246f6b --- /dev/null +++ b/worker/job.go @@ -0,0 +1,202 @@ +package worker + +import ( + "errors" + "time" +) + +// this file contains the workflow of a mirror jb + +type ctrlAction uint8 + +const ( + jobStart ctrlAction = iota + jobStop // stop syncing keep the job + jobDisable // disable the job (stops goroutine) + jobRestart // restart syncing + jobPing // ensure the goroutine is alive +) + +// runMirrorJob is the goroutine where syncing job runs in +// arguments: +// provider: mirror provider object +// ctrlChan: receives messages from the manager +// managerChan: push messages to the manager +// sempaphore: make sure the concurrent running syncing job won't explode +// TODO: message struct for managerChan +func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerChan chan<- struct{}, semaphore chan empty) error { + + // to make code shorter + runHooks := func(Hooks []jobHook, action func(h jobHook) error, hookname string) error { + for _, hook := range Hooks { + if err := action(hook); err != nil { + logger.Error( + "failed at %s hooks for %s: %s", + hookname, provider.Name(), err.Error(), + ) + return err + } + } + return nil + } + + runJobWrapper := func(kill <-chan empty, jobDone chan<- empty) error { + defer func() { jobDone <- empty{} }() + logger.Info("start syncing: %s", provider.Name()) + + Hooks := provider.Hooks() + rHooks := []jobHook{} + for i := len(Hooks); i > 0; i-- { + rHooks = append(rHooks, Hooks[i-1]) + } + + logger.Debug("hooks: pre-job") + err := runHooks(Hooks, func(h jobHook) error { return h.preJob() }, "pre-job") + if err != nil { + return err + } + + for retry := 0; retry < maxRetry; retry++ { + stopASAP := false // stop job as soon as possible + + if retry > 0 { + logger.Info("retry syncing: %s, retry: %d", provider.Name(), retry) + } + err := runHooks(Hooks, func(h jobHook) error { return h.preExec() }, "pre-exec") + if err != nil { + return err + } + + // start syncing + err = provider.Start() + if err != nil { + logger.Error( + "failed to start syncing job for %s: %s", + provider.Name(), err.Error(), + ) + return err + } + var syncErr error + syncDone := make(chan error, 1) + go func() { + err := provider.Wait() + if !stopASAP { + syncDone <- err + } + }() + + select { + case syncErr = <-syncDone: + logger.Debug("syncing done") + case <-kill: + stopASAP = true + err := provider.Terminate() + if err != nil { + logger.Error("failed to terminate provider %s: %s", provider.Name(), err.Error()) + return err + } + syncErr = errors.New("killed by manager") + } + + // post-exec hooks + herr := runHooks(rHooks, func(h jobHook) error { return h.postExec() }, "post-exec") + if herr != nil { + return herr + } + + if syncErr == nil { + // syncing success + logger.Info("succeeded syncing %s", provider.Name()) + managerChan <- struct{}{} + // post-success hooks + err := runHooks(rHooks, func(h jobHook) error { return h.postSuccess() }, "post-success") + if err != nil { + return err + } + return nil + + } + + // syncing failed + logger.Info("failed syncing %s: %s", provider.Name(), err.Error()) + managerChan <- struct{}{} + // post-fail hooks + err = runHooks(rHooks, func(h jobHook) error { return h.postFail() }, "post-fail") + if err != nil { + return err + } + // gracefully exit + if stopASAP { + return nil + } + // continue to next retry + } // for retry + return nil + } + + runJob := func(kill <-chan empty, jobDone chan<- empty) { + select { + case <-semaphore: + defer func() { semaphore <- empty{} }() + runJobWrapper(kill, jobDone) + case <-kill: + return + } + } + + enabled := true // whether this job is stopped by the manager + for { + if enabled { + kill := make(chan empty) + jobDone := make(chan empty) + go runJob(kill, jobDone) + + _wait_for_job: + select { + case <-jobDone: + logger.Debug("job done") + case ctrl := <-ctrlChan: + switch ctrl { + case jobStop: + enabled = false + close(kill) + case jobDisable: + close(kill) + return nil + case jobRestart: + enabled = true + close(kill) + continue + case jobStart: + enabled = true + goto _wait_for_job + default: + // TODO: implement this + close(kill) + return nil + } + } + } + + select { + case <-time.After(provider.Interval()): + continue + case ctrl := <-ctrlChan: + switch ctrl { + case jobStop: + enabled = false + case jobDisable: + return nil + case jobRestart: + enabled = true + case jobStart: + enabled = true + default: + // TODO + return nil + } + } + } + + return nil +} diff --git a/worker/job_test.go b/worker/job_test.go new file mode 100644 index 0000000..2388af1 --- /dev/null +++ b/worker/job_test.go @@ -0,0 +1,135 @@ +package worker + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestMirrorJob(t *testing.T) { + + Convey("MirrorJob should work", t, func(ctx C) { + tmpDir, err := ioutil.TempDir("", "tunasync") + defer os.RemoveAll(tmpDir) + So(err, ShouldBeNil) + scriptFile := filepath.Join(tmpDir, "cmd.sh") + tmpFile := filepath.Join(tmpDir, "log_file") + + c := cmdConfig{ + name: "tuna-cmd-jobtest", + upstreamURL: "http://mirrors.tuna.moe/", + command: "bash " + scriptFile, + workingDir: tmpDir, + logDir: tmpDir, + logFile: tmpFile, + interval: 1 * time.Second, + } + + provider, err := newCmdProvider(c) + So(err, ShouldBeNil) + + So(provider.Name(), ShouldEqual, c.name) + So(provider.WorkingDir(), ShouldEqual, c.workingDir) + So(provider.LogDir(), ShouldEqual, c.logDir) + So(provider.LogFile(), ShouldEqual, c.logFile) + So(provider.Interval(), ShouldEqual, c.interval) + + Convey("For a normal mirror job", func(ctx C) { + scriptContent := `#!/bin/bash + echo $TUNASYNC_WORKING_DIR + echo $TUNASYNC_MIRROR_NAME + echo $TUNASYNC_UPSTREAM_URL + echo $TUNASYNC_LOG_FILE + ` + exceptedOutput := fmt.Sprintf( + "%s\n%s\n%s\n%s\n", + provider.WorkingDir(), + provider.Name(), + provider.upstreamURL, + provider.LogFile(), + ) + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + readedScriptContent, err := ioutil.ReadFile(scriptFile) + So(err, ShouldBeNil) + So(readedScriptContent, ShouldResemble, []byte(scriptContent)) + + Convey("If we let it run several times", func(ctx C) { + ctrlChan := make(chan ctrlAction) + managerChan := make(chan struct{}) + semaphore := make(chan empty, 1) + semaphore <- empty{} + + go runMirrorJob(provider, ctrlChan, managerChan, semaphore) + for i := 0; i < 2; i++ { + <-managerChan + loggedContent, err := ioutil.ReadFile(provider.LogFile()) + So(err, ShouldBeNil) + So(string(loggedContent), ShouldEqual, exceptedOutput) + } + select { + case <-managerChan: + So(0, ShouldEqual, 0) // made this fail + case <-time.After(2 * time.Second): + So(0, ShouldEqual, 1) + } + ctrlChan <- jobDisable + select { + case <-managerChan: + So(0, ShouldEqual, 1) // made this fail + case <-time.After(2 * time.Second): + So(0, ShouldEqual, 0) + } + }) + + }) + + Convey("When running long jobs", func(ctx C) { + scriptContent := `#!/bin/bash +echo $TUNASYNC_WORKING_DIR +sleep 5 +echo $TUNASYNC_WORKING_DIR + ` + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + + ctrlChan := make(chan ctrlAction) + managerChan := make(chan struct{}) + semaphore := make(chan empty, 1) + semaphore <- empty{} + + Convey("If we kill it", func(ctx C) { + go runMirrorJob(provider, ctrlChan, managerChan, semaphore) + time.Sleep(1 * time.Second) + ctrlChan <- jobStop + time.Sleep(1 * time.Second) + exceptedOutput := fmt.Sprintf("%s\n", provider.WorkingDir()) + loggedContent, err := ioutil.ReadFile(provider.LogFile()) + So(err, ShouldBeNil) + So(string(loggedContent), ShouldEqual, exceptedOutput) + ctrlChan <- jobDisable + }) + Convey("If we don't kill it", func(ctx C) { + go runMirrorJob(provider, ctrlChan, managerChan, semaphore) + <-managerChan + + exceptedOutput := fmt.Sprintf( + "%s\n%s\n", + provider.WorkingDir(), provider.WorkingDir(), + ) + + loggedContent, err := ioutil.ReadFile(provider.LogFile()) + So(err, ShouldBeNil) + So(string(loggedContent), ShouldEqual, exceptedOutput) + ctrlChan <- jobDisable + }) + }) + + }) + +} diff --git a/worker/provider.go b/worker/provider.go index 806c073..7aca550 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -1,5 +1,7 @@ package worker +import "time" + // mirror provider is the wrapper of mirror jobs type providerType uint8 @@ -17,15 +19,15 @@ type mirrorProvider interface { // TODO: implement Run, Terminate and Hooks // run mirror job in background - Start() + Start() error // Wait job to finish - Wait() + Wait() error // terminate mirror job - Terminate() + Terminate() error // job hooks - Hooks() + Hooks() []jobHook - Interval() int + Interval() time.Duration WorkingDir() string LogDir() string @@ -42,7 +44,8 @@ type mirrorProvider interface { type baseProvider struct { ctx *Context name string - interval int + interval time.Duration + hooks []jobHook } func (p *baseProvider) Name() string { @@ -63,7 +66,7 @@ func (p *baseProvider) Context() *Context { return p.ctx } -func (p *baseProvider) Interval() int { +func (p *baseProvider) Interval() time.Duration { return p.interval } @@ -93,3 +96,11 @@ func (p *baseProvider) LogFile() string { } panic("log dir is impossible to be unavailable") } + +func (p *baseProvider) AddHook(hook jobHook) { + p.hooks = append(p.hooks, hook) +} + +func (p *baseProvider) Hooks() []jobHook { + return p.hooks +} diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go index 1687109..ba8dcf1 100644 --- a/worker/rsync_provider.go +++ b/worker/rsync_provider.go @@ -1,11 +1,13 @@ package worker +import "time" + type rsyncConfig struct { name string upstreamURL, password, excludeFile string workingDir, logDir, logFile string useIPv6 bool - interval int + interval time.Duration } // An RsyncProvider provides the implementation to rsync-based syncing jobs @@ -41,8 +43,3 @@ func (p *rsyncProvider) Start() { func (p *rsyncProvider) Terminate() { } - -// TODO: implement this -func (p *rsyncProvider) Hooks() { - -} diff --git a/worker/runner.go b/worker/runner.go index 92edbba..ed6bc65 100644 --- a/worker/runner.go +++ b/worker/runner.go @@ -20,7 +20,7 @@ type cmdJob struct { workingDir string env map[string]string logFile *os.File - finished chan struct{} + finished chan empty } func newCmdJob(cmdAndArgs []string, workingDir string, env map[string]string) *cmdJob { @@ -46,13 +46,13 @@ func newCmdJob(cmdAndArgs []string, workingDir string, env map[string]string) *c } func (c *cmdJob) Start() error { - c.finished = make(chan struct{}, 1) + c.finished = make(chan empty, 1) return c.cmd.Start() } func (c *cmdJob) Wait() error { err := c.cmd.Wait() - c.finished <- struct{}{} + c.finished <- empty{} return err } @@ -63,10 +63,10 @@ func (c *cmdJob) SetLogFile(logFile *os.File) { func (c *cmdJob) Terminate() error { if c.cmd == nil { - return errors.New("Command not initialized") + return nil } if c.cmd.Process == nil { - return errors.New("No Process Running") + return nil } err := unix.Kill(c.cmd.Process.Pid, syscall.SIGTERM) if err != nil { From afee5b2a819bb9d7d521f9b574b084be2ff49986 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sat, 23 Apr 2016 21:54:00 +0800 Subject: [PATCH 18/66] feature(manager): skeleton for worker-manager communication --- internal/msg.go | 42 ++++++++++++++++++++++++++++++++++++++++++ manager/db.go | 23 +++++++++++++++++++++++ manager/server.go | 12 +++++++++--- 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 internal/msg.go create mode 100644 manager/db.go diff --git a/internal/msg.go b/internal/msg.go new file mode 100644 index 0000000..5ff8ce9 --- /dev/null +++ b/internal/msg.go @@ -0,0 +1,42 @@ +package internal + +import "time" + +// A StatusUpdateMsg represents a msg when +// a worker has done syncing +type StatusUpdateMsg struct { + Name string `json:"name"` + Worker string `json:"worker"` + IsMaster bool `json:"is_master"` + Status SyncStatus `json:"status"` + LastUpdate time.Time `json:"last_update"` + Upstream string `json:"upstream"` + Size string `json:"size"` +} + +// A WorkerInfoMsg is +type WorkerInfoMsg struct { + Name string `json:"name"` +} + +type CmdVerb uint8 + +const ( + CmdStart CmdVerb = iota + CmdStop // stop syncing keep the job + CmdDisable // disable the job (stops goroutine) + CmdRestart // restart syncing + CmdPing // ensure the goroutine is alive +) + +type WorkerCmd struct { + Cmd CmdVerb `json:"cmd"` + Args []string `json:"args"` +} + +type ClientCmd struct { + Cmd CmdVerb `json:"cmd"` + MirrorID string `json:"mirror_id"` + WorkerID string `json:"worker_id"` + Args []string `json:"args"` +} diff --git a/manager/db.go b/manager/db.go new file mode 100644 index 0000000..4cf3edd --- /dev/null +++ b/manager/db.go @@ -0,0 +1,23 @@ +package manager + +import "github.com/boltdb/bolt" + +type dbAdapter interface { + GetWorker(workerID string) + UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) + GetMirrorStatus(workerID, mirrorID string) + GetMirrorStatusList(workerID string) + Close() +} + +type boltAdapter struct { + db *bolt.DB + dbFile string +} + +func (b *boltAdapter) Close() error { + if b.db != nil { + return b.db.Close() + } + return nil +} diff --git a/manager/server.go b/manager/server.go index 3c41889..4d674bd 100644 --- a/manager/server.go +++ b/manager/server.go @@ -23,14 +23,20 @@ func makeHTTPServer(debug bool) *gin.Engine { r.GET("/ping", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"msg": "pong"}) }) - // List jobs, status page + // list jobs, status page r.GET("/jobs", func(c *gin.Context) {}) // worker online r.POST("/workers/:name", func(c *gin.Context) {}) - // post job list - r.POST("/workers/:name/jobs", func(c *gin.Context) {}) + // get job list + r.GET("/workers/:name/jobs", func(c *gin.Context) {}) // post job status r.POST("/workers/:name/jobs/:job", func(c *gin.Context) {}) + // worker command polling + r.GET("/workers/:name/cmd_stream", func(c *gin.Context) {}) + + // for tunasynctl to post commands + r.POST("/cmd/", func(c *gin.Context) {}) + return r } From 6b05a5894e7b5aa4810cbc57b4edc1be2d9557a3 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 24 Apr 2016 16:48:47 +0800 Subject: [PATCH 19/66] feature(worker): runMirrorJob no longer controls the interval --- internal/logger.go | 4 ++-- worker/job.go | 50 +++++++++++++++++++++++----------------------- worker/job_test.go | 6 +++++- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/internal/logger.go b/internal/logger.go index bf1a978..9ac0632 100644 --- a/internal/logger.go +++ b/internal/logger.go @@ -10,9 +10,9 @@ import ( func InitLogger(verbose, debug, withSystemd bool) { var fmtString string if withSystemd { - fmtString = "\r[%{level:.6s}] %{message}" + fmtString = "[%{level:.6s}] %{message}" } else { - fmtString = "\r%{color}[%{time:06-01-02 15:04:05}][%{level:.6s}]%{color:reset} %{message}" + fmtString = "%{color}[%{time:06-01-02 15:04:05}][%{level:.6s}][%{shortfile}]%{color:reset} %{message}" } format := logging.MustStringFormatter(fmtString) logging.SetFormatter(format) diff --git a/worker/job.go b/worker/job.go index 5246f6b..fe459f3 100644 --- a/worker/job.go +++ b/worker/job.go @@ -1,9 +1,6 @@ package worker -import ( - "errors" - "time" -) +import "errors" // this file contains the workflow of a mirror jb @@ -41,7 +38,8 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh } runJobWrapper := func(kill <-chan empty, jobDone chan<- empty) error { - defer func() { jobDone <- empty{} }() + defer close(jobDone) + logger.Info("start syncing: %s", provider.Name()) Hooks := provider.Hooks() @@ -89,6 +87,7 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh case syncErr = <-syncDone: logger.Debug("syncing done") case <-kill: + logger.Debug("received kill") stopASAP = true err := provider.Terminate() if err != nil { @@ -118,15 +117,18 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh } // syncing failed - logger.Info("failed syncing %s: %s", provider.Name(), err.Error()) + logger.Warning("failed syncing %s: %s", provider.Name(), syncErr.Error()) managerChan <- struct{}{} + // post-fail hooks + logger.Debug("post-fail hooks") err = runHooks(rHooks, func(h jobHook) error { return h.postFail() }, "post-fail") if err != nil { return err } // gracefully exit if stopASAP { + logger.Debug("No retry, exit directly") return nil } // continue to next retry @@ -140,6 +142,7 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh defer func() { semaphore <- empty{} }() runJobWrapper(kill, jobDone) case <-kill: + jobDone <- empty{} return } } @@ -160,12 +163,15 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh case jobStop: enabled = false close(kill) + <-jobDone case jobDisable: close(kill) + <-jobDone return nil case jobRestart: enabled = true close(kill) + <-jobDone continue case jobStart: enabled = true @@ -178,25 +184,19 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh } } - select { - case <-time.After(provider.Interval()): - continue - case ctrl := <-ctrlChan: - switch ctrl { - case jobStop: - enabled = false - case jobDisable: - return nil - case jobRestart: - enabled = true - case jobStart: - enabled = true - default: - // TODO - return nil - } + ctrl := <-ctrlChan + switch ctrl { + case jobStop: + enabled = false + case jobDisable: + return nil + case jobRestart: + enabled = true + case jobStart: + enabled = true + default: + // TODO + return nil } } - - return nil } diff --git a/worker/job_test.go b/worker/job_test.go index 2388af1..0fa90e4 100644 --- a/worker/job_test.go +++ b/worker/job_test.go @@ -9,10 +9,13 @@ import ( "time" . "github.com/smartystreets/goconvey/convey" + . "github.com/tuna/tunasync/internal" ) func TestMirrorJob(t *testing.T) { + InitLogger(true, true, false) + Convey("MirrorJob should work", t, func(ctx C) { tmpDir, err := ioutil.TempDir("", "tunasync") defer os.RemoveAll(tmpDir) @@ -71,6 +74,7 @@ func TestMirrorJob(t *testing.T) { loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) So(string(loggedContent), ShouldEqual, exceptedOutput) + ctrlChan <- jobStart } select { case <-managerChan: @@ -107,7 +111,7 @@ echo $TUNASYNC_WORKING_DIR go runMirrorJob(provider, ctrlChan, managerChan, semaphore) time.Sleep(1 * time.Second) ctrlChan <- jobStop - time.Sleep(1 * time.Second) + <-managerChan exceptedOutput := fmt.Sprintf("%s\n", provider.WorkingDir()) loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) From 26b7ef9a9c02b5eb364c7d899be1b5e19e4d2e26 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 24 Apr 2016 17:02:49 +0800 Subject: [PATCH 20/66] refactor(worker): use write blocking for semaphore --- worker/job.go | 4 ++-- worker/job_test.go | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/worker/job.go b/worker/job.go index fe459f3..b93844f 100644 --- a/worker/job.go +++ b/worker/job.go @@ -138,8 +138,8 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh runJob := func(kill <-chan empty, jobDone chan<- empty) { select { - case <-semaphore: - defer func() { semaphore <- empty{} }() + case semaphore <- empty{}: + defer func() { <-semaphore }() runJobWrapper(kill, jobDone) case <-kill: jobDone <- empty{} diff --git a/worker/job_test.go b/worker/job_test.go index 0fa90e4..f9ab7ee 100644 --- a/worker/job_test.go +++ b/worker/job_test.go @@ -66,7 +66,6 @@ func TestMirrorJob(t *testing.T) { ctrlChan := make(chan ctrlAction) managerChan := make(chan struct{}) semaphore := make(chan empty, 1) - semaphore <- empty{} go runMirrorJob(provider, ctrlChan, managerChan, semaphore) for i := 0; i < 2; i++ { @@ -105,7 +104,6 @@ echo $TUNASYNC_WORKING_DIR ctrlChan := make(chan ctrlAction) managerChan := make(chan struct{}) semaphore := make(chan empty, 1) - semaphore <- empty{} Convey("If we kill it", func(ctx C) { go runMirrorJob(provider, ctrlChan, managerChan, semaphore) From f31bcfbcc365dde5b3b537d7940b4f8ec073759d Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 24 Apr 2016 17:20:47 +0800 Subject: [PATCH 21/66] feature(API): error message in manager channel --- internal/msg.go | 1 + worker/job.go | 27 ++++++++++++++++++++++----- worker/job_test.go | 40 +++++++++++++++++++++++++++++++++------- 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/internal/msg.go b/internal/msg.go index 5ff8ce9..d34116f 100644 --- a/internal/msg.go +++ b/internal/msg.go @@ -12,6 +12,7 @@ type StatusUpdateMsg struct { LastUpdate time.Time `json:"last_update"` Upstream string `json:"upstream"` Size string `json:"size"` + ErrorMsg string `json:"error_msg"` } // A WorkerInfoMsg is diff --git a/worker/job.go b/worker/job.go index b93844f..cae589a 100644 --- a/worker/job.go +++ b/worker/job.go @@ -1,6 +1,11 @@ package worker -import "errors" +import ( + "errors" + "fmt" + + tunasync "github.com/tuna/tunasync/internal" +) // this file contains the workflow of a mirror jb @@ -14,14 +19,20 @@ const ( jobPing // ensure the goroutine is alive ) +type jobMessage struct { + status tunasync.SyncStatus + name string + msg string +} + // runMirrorJob is the goroutine where syncing job runs in // arguments: // provider: mirror provider object // ctrlChan: receives messages from the manager -// managerChan: push messages to the manager +// managerChan: push messages to the manager, this channel should have a larger buffer // sempaphore: make sure the concurrent running syncing job won't explode // TODO: message struct for managerChan -func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerChan chan<- struct{}, semaphore chan empty) error { +func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerChan chan<- jobMessage, semaphore chan empty) error { // to make code shorter runHooks := func(Hooks []jobHook, action func(h jobHook) error, hookname string) error { @@ -31,6 +42,10 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh "failed at %s hooks for %s: %s", hookname, provider.Name(), err.Error(), ) + managerChan <- jobMessage{ + tunasync.Failed, provider.Name(), + fmt.Sprintf("error exec hook %s: %s", hookname, err.Error()), + } return err } } @@ -40,6 +55,7 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh runJobWrapper := func(kill <-chan empty, jobDone chan<- empty) error { defer close(jobDone) + managerChan <- jobMessage{tunasync.PreSyncing, provider.Name(), ""} logger.Info("start syncing: %s", provider.Name()) Hooks := provider.Hooks() @@ -66,6 +82,7 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh } // start syncing + managerChan <- jobMessage{tunasync.Syncing, provider.Name(), ""} err = provider.Start() if err != nil { logger.Error( @@ -106,7 +123,7 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh if syncErr == nil { // syncing success logger.Info("succeeded syncing %s", provider.Name()) - managerChan <- struct{}{} + managerChan <- jobMessage{tunasync.Success, provider.Name(), ""} // post-success hooks err := runHooks(rHooks, func(h jobHook) error { return h.postSuccess() }, "post-success") if err != nil { @@ -118,7 +135,7 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh // syncing failed logger.Warning("failed syncing %s: %s", provider.Name(), syncErr.Error()) - managerChan <- struct{}{} + managerChan <- jobMessage{tunasync.Failed, provider.Name(), syncErr.Error()} // post-fail hooks logger.Debug("post-fail hooks") diff --git a/worker/job_test.go b/worker/job_test.go index f9ab7ee..f5e9382 100644 --- a/worker/job_test.go +++ b/worker/job_test.go @@ -64,23 +64,34 @@ func TestMirrorJob(t *testing.T) { Convey("If we let it run several times", func(ctx C) { ctrlChan := make(chan ctrlAction) - managerChan := make(chan struct{}) + managerChan := make(chan jobMessage, 10) semaphore := make(chan empty, 1) go runMirrorJob(provider, ctrlChan, managerChan, semaphore) for i := 0; i < 2; i++ { - <-managerChan + msg := <-managerChan + So(msg.status, ShouldEqual, PreSyncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Syncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Success) loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) So(string(loggedContent), ShouldEqual, exceptedOutput) ctrlChan <- jobStart } select { - case <-managerChan: - So(0, ShouldEqual, 0) // made this fail + case msg := <-managerChan: + So(msg.status, ShouldEqual, PreSyncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Syncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Success) + case <-time.After(2 * time.Second): So(0, ShouldEqual, 1) } + ctrlChan <- jobDisable select { case <-managerChan: @@ -102,23 +113,38 @@ echo $TUNASYNC_WORKING_DIR So(err, ShouldBeNil) ctrlChan := make(chan ctrlAction) - managerChan := make(chan struct{}) + managerChan := make(chan jobMessage, 10) semaphore := make(chan empty, 1) Convey("If we kill it", func(ctx C) { go runMirrorJob(provider, ctrlChan, managerChan, semaphore) + time.Sleep(1 * time.Second) + msg := <-managerChan + So(msg.status, ShouldEqual, PreSyncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Syncing) + ctrlChan <- jobStop - <-managerChan + + msg = <-managerChan + So(msg.status, ShouldEqual, Failed) + exceptedOutput := fmt.Sprintf("%s\n", provider.WorkingDir()) loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) So(string(loggedContent), ShouldEqual, exceptedOutput) ctrlChan <- jobDisable }) + Convey("If we don't kill it", func(ctx C) { go runMirrorJob(provider, ctrlChan, managerChan, semaphore) - <-managerChan + msg := <-managerChan + So(msg.status, ShouldEqual, PreSyncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Syncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Success) exceptedOutput := fmt.Sprintf( "%s\n%s\n", From b077db1d0bbecb771eb63ed142b882bb8fd395b2 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 24 Apr 2016 20:23:44 +0800 Subject: [PATCH 22/66] feature(worker): job schedule --- worker/job.go | 65 ++++++++++++++++++++++++------------- worker/job_test.go | 21 ++++++------ worker/main.go | 17 ++++++++++ worker/schedule.go | 71 +++++++++++++++++++++++++++++++++++++++++ worker/schedule_test.go | 50 +++++++++++++++++++++++++++++ 5 files changed, 191 insertions(+), 33 deletions(-) create mode 100644 worker/main.go create mode 100644 worker/schedule.go create mode 100644 worker/schedule_test.go diff --git a/worker/job.go b/worker/job.go index cae589a..50d4c3d 100644 --- a/worker/job.go +++ b/worker/job.go @@ -25,6 +25,24 @@ type jobMessage struct { msg string } +type mirrorJob struct { + provider mirrorProvider + ctrlChan chan ctrlAction + enabled bool +} + +func newMirrorJob(provider mirrorProvider) *mirrorJob { + return &mirrorJob{ + provider: provider, + ctrlChan: make(chan ctrlAction, 1), + enabled: true, + } +} + +func (m *mirrorJob) Name() string { + return m.provider.Name() +} + // runMirrorJob is the goroutine where syncing job runs in // arguments: // provider: mirror provider object @@ -32,7 +50,9 @@ type jobMessage struct { // managerChan: push messages to the manager, this channel should have a larger buffer // sempaphore: make sure the concurrent running syncing job won't explode // TODO: message struct for managerChan -func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerChan chan<- jobMessage, semaphore chan empty) error { +func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) error { + + provider := m.provider // to make code shorter runHooks := func(Hooks []jobHook, action func(h jobHook) error, hookname string) error { @@ -40,10 +60,10 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh if err := action(hook); err != nil { logger.Error( "failed at %s hooks for %s: %s", - hookname, provider.Name(), err.Error(), + hookname, m.Name(), err.Error(), ) managerChan <- jobMessage{ - tunasync.Failed, provider.Name(), + tunasync.Failed, m.Name(), fmt.Sprintf("error exec hook %s: %s", hookname, err.Error()), } return err @@ -55,8 +75,8 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh runJobWrapper := func(kill <-chan empty, jobDone chan<- empty) error { defer close(jobDone) - managerChan <- jobMessage{tunasync.PreSyncing, provider.Name(), ""} - logger.Info("start syncing: %s", provider.Name()) + managerChan <- jobMessage{tunasync.PreSyncing, m.Name(), ""} + logger.Info("start syncing: %s", m.Name()) Hooks := provider.Hooks() rHooks := []jobHook{} @@ -74,7 +94,7 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh stopASAP := false // stop job as soon as possible if retry > 0 { - logger.Info("retry syncing: %s, retry: %d", provider.Name(), retry) + logger.Info("retry syncing: %s, retry: %d", m.Name(), retry) } err := runHooks(Hooks, func(h jobHook) error { return h.preExec() }, "pre-exec") if err != nil { @@ -82,12 +102,12 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh } // start syncing - managerChan <- jobMessage{tunasync.Syncing, provider.Name(), ""} + managerChan <- jobMessage{tunasync.Syncing, m.Name(), ""} err = provider.Start() if err != nil { logger.Error( "failed to start syncing job for %s: %s", - provider.Name(), err.Error(), + m.Name(), err.Error(), ) return err } @@ -108,7 +128,7 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh stopASAP = true err := provider.Terminate() if err != nil { - logger.Error("failed to terminate provider %s: %s", provider.Name(), err.Error()) + logger.Error("failed to terminate provider %s: %s", m.Name(), err.Error()) return err } syncErr = errors.New("killed by manager") @@ -122,8 +142,8 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh if syncErr == nil { // syncing success - logger.Info("succeeded syncing %s", provider.Name()) - managerChan <- jobMessage{tunasync.Success, provider.Name(), ""} + logger.Info("succeeded syncing %s", m.Name()) + managerChan <- jobMessage{tunasync.Success, m.Name(), ""} // post-success hooks err := runHooks(rHooks, func(h jobHook) error { return h.postSuccess() }, "post-success") if err != nil { @@ -134,8 +154,8 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh } // syncing failed - logger.Warning("failed syncing %s: %s", provider.Name(), syncErr.Error()) - managerChan <- jobMessage{tunasync.Failed, provider.Name(), syncErr.Error()} + logger.Warning("failed syncing %s: %s", m.Name(), syncErr.Error()) + managerChan <- jobMessage{tunasync.Failed, m.Name(), syncErr.Error()} // post-fail hooks logger.Debug("post-fail hooks") @@ -164,9 +184,8 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh } } - enabled := true // whether this job is stopped by the manager for { - if enabled { + if m.enabled { kill := make(chan empty) jobDone := make(chan empty) go runJob(kill, jobDone) @@ -175,10 +194,10 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh select { case <-jobDone: logger.Debug("job done") - case ctrl := <-ctrlChan: + case ctrl := <-m.ctrlChan: switch ctrl { case jobStop: - enabled = false + m.enabled = false close(kill) <-jobDone case jobDisable: @@ -186,12 +205,12 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh <-jobDone return nil case jobRestart: - enabled = true + m.enabled = true close(kill) <-jobDone continue case jobStart: - enabled = true + m.enabled = true goto _wait_for_job default: // TODO: implement this @@ -201,16 +220,16 @@ func runMirrorJob(provider mirrorProvider, ctrlChan <-chan ctrlAction, managerCh } } - ctrl := <-ctrlChan + ctrl := <-m.ctrlChan switch ctrl { case jobStop: - enabled = false + m.enabled = false case jobDisable: return nil case jobRestart: - enabled = true + m.enabled = true case jobStart: - enabled = true + m.enabled = true default: // TODO return nil diff --git a/worker/job_test.go b/worker/job_test.go index f5e9382..065d10e 100644 --- a/worker/job_test.go +++ b/worker/job_test.go @@ -63,11 +63,11 @@ func TestMirrorJob(t *testing.T) { So(readedScriptContent, ShouldResemble, []byte(scriptContent)) Convey("If we let it run several times", func(ctx C) { - ctrlChan := make(chan ctrlAction) managerChan := make(chan jobMessage, 10) semaphore := make(chan empty, 1) + job := newMirrorJob(provider) - go runMirrorJob(provider, ctrlChan, managerChan, semaphore) + go job.Run(managerChan, semaphore) for i := 0; i < 2; i++ { msg := <-managerChan So(msg.status, ShouldEqual, PreSyncing) @@ -78,7 +78,7 @@ func TestMirrorJob(t *testing.T) { loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) So(string(loggedContent), ShouldEqual, exceptedOutput) - ctrlChan <- jobStart + job.ctrlChan <- jobStart } select { case msg := <-managerChan: @@ -92,7 +92,7 @@ func TestMirrorJob(t *testing.T) { So(0, ShouldEqual, 1) } - ctrlChan <- jobDisable + job.ctrlChan <- jobDisable select { case <-managerChan: So(0, ShouldEqual, 1) // made this fail @@ -112,12 +112,12 @@ echo $TUNASYNC_WORKING_DIR err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) So(err, ShouldBeNil) - ctrlChan := make(chan ctrlAction) managerChan := make(chan jobMessage, 10) semaphore := make(chan empty, 1) + job := newMirrorJob(provider) Convey("If we kill it", func(ctx C) { - go runMirrorJob(provider, ctrlChan, managerChan, semaphore) + go job.Run(managerChan, semaphore) time.Sleep(1 * time.Second) msg := <-managerChan @@ -125,7 +125,7 @@ echo $TUNASYNC_WORKING_DIR msg = <-managerChan So(msg.status, ShouldEqual, Syncing) - ctrlChan <- jobStop + job.ctrlChan <- jobStop msg = <-managerChan So(msg.status, ShouldEqual, Failed) @@ -134,11 +134,12 @@ echo $TUNASYNC_WORKING_DIR loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) So(string(loggedContent), ShouldEqual, exceptedOutput) - ctrlChan <- jobDisable + job.ctrlChan <- jobDisable }) Convey("If we don't kill it", func(ctx C) { - go runMirrorJob(provider, ctrlChan, managerChan, semaphore) + go job.Run(managerChan, semaphore) + msg := <-managerChan So(msg.status, ShouldEqual, PreSyncing) msg = <-managerChan @@ -154,7 +155,7 @@ echo $TUNASYNC_WORKING_DIR loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) So(string(loggedContent), ShouldEqual, exceptedOutput) - ctrlChan <- jobDisable + job.ctrlChan <- jobDisable }) }) diff --git a/worker/main.go b/worker/main.go new file mode 100644 index 0000000..b1d1453 --- /dev/null +++ b/worker/main.go @@ -0,0 +1,17 @@ +package worker + +import "time" + +// toplevel module for workers + +func main() { + + for { + // if time.Now().After() { + // + // } + + time.Sleep(1 * time.Second) + } + +} diff --git a/worker/schedule.go b/worker/schedule.go new file mode 100644 index 0000000..cb95a5d --- /dev/null +++ b/worker/schedule.go @@ -0,0 +1,71 @@ +package worker + +// schedule queue for jobs + +import ( + "sync" + "time" + + "github.com/ryszard/goskiplist/skiplist" +) + +type scheduleQueue struct { + sync.Mutex + list *skiplist.SkipList +} + +func timeLessThan(l, r interface{}) bool { + tl := l.(time.Time) + tr := r.(time.Time) + return tl.Before(tr) +} + +func newScheduleQueue() *scheduleQueue { + queue := new(scheduleQueue) + queue.list = skiplist.NewCustomMap(timeLessThan) + return queue +} + +func (q *scheduleQueue) AddJob(schedTime time.Time, job *mirrorJob) { + q.Lock() + defer q.Unlock() + q.list.Set(schedTime, job) +} + +// pop out the first job if it's time to run it +func (q *scheduleQueue) Pop() *mirrorJob { + q.Lock() + defer q.Unlock() + + first := q.list.SeekToFirst() + if first == nil { + return nil + } + defer first.Close() + + t := first.Key().(time.Time) + if t.Before(time.Now()) { + job := first.Value().(*mirrorJob) + q.list.Delete(first.Key()) + return job + } + return nil +} + +// remove job +func (q *scheduleQueue) Remove(name string) bool { + q.Lock() + defer q.Unlock() + + cur := q.list.Iterator() + defer cur.Close() + + for cur.Next() { + cj := cur.Value().(*mirrorJob) + if cj.Name() == name { + q.list.Delete(cur.Key()) + return true + } + } + return false +} diff --git a/worker/schedule_test.go b/worker/schedule_test.go new file mode 100644 index 0000000..8bf3bc5 --- /dev/null +++ b/worker/schedule_test.go @@ -0,0 +1,50 @@ +package worker + +import ( + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestSchedule(t *testing.T) { + + Convey("MirrorJobSchedule should work", t, func(ctx C) { + schedule := newScheduleQueue() + + Convey("When poping on empty schedule", func() { + job := schedule.Pop() + So(job, ShouldBeNil) + }) + + Convey("When adding some jobs", func() { + c := cmdConfig{ + name: "schedule_test", + } + provider, _ := newCmdProvider(c) + job := newMirrorJob(provider) + sched := time.Now().Add(1 * time.Second) + + schedule.AddJob(sched, job) + So(schedule.Pop(), ShouldBeNil) + time.Sleep(1200 * time.Millisecond) + So(schedule.Pop(), ShouldEqual, job) + + }) + Convey("When removing jobs", func() { + c := cmdConfig{ + name: "schedule_test", + } + provider, _ := newCmdProvider(c) + job := newMirrorJob(provider) + sched := time.Now().Add(1 * time.Second) + + schedule.AddJob(sched, job) + So(schedule.Remove("something"), ShouldBeFalse) + So(schedule.Remove("schedule_test"), ShouldBeTrue) + time.Sleep(1200 * time.Millisecond) + So(schedule.Pop(), ShouldBeNil) + }) + + }) +} From d8b45d7231cc03d23df0f053d915ff0894416bae Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 24 Apr 2016 21:13:11 +0800 Subject: [PATCH 23/66] feature(worker): worker config file --- worker/config.go | 90 +++++++++++++++++++++++++++++++++++++++++++ worker/config_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 worker/config.go create mode 100644 worker/config_test.go diff --git a/worker/config.go b/worker/config.go new file mode 100644 index 0000000..a2fe1bd --- /dev/null +++ b/worker/config.go @@ -0,0 +1,90 @@ +package worker + +import ( + "errors" + "os" + + "github.com/BurntSushi/toml" +) + +type ProviderEnum uint8 + +const ( + ProvRsync ProviderEnum = iota + ProvTwoStageRsync + ProvCommand +) + +func (p ProviderEnum) MarshalText() ([]byte, error) { + + switch p { + case ProvCommand: + return []byte("command"), nil + case ProvRsync: + return []byte("rsync"), nil + case ProvTwoStageRsync: + return []byte("two-stage-rsync"), nil + default: + return []byte{}, errors.New("Invalid ProviderEnum value") + } + +} + +func (p *ProviderEnum) UnmarshalText(text []byte) error { + s := string(text) + switch s { + case `command`: + *p = ProvCommand + case `rsync`: + *p = ProvRsync + case `two-stage-rsync`: + *p = ProvTwoStageRsync + default: + return errors.New("Invalid value to provierEnum") + } + return nil + +} + +type Config struct { + Global globalConfig `toml:"global"` + Mirrors []mirrorConfig `toml:"mirrors"` +} + +type globalConfig struct { + Name string `toml:"name"` + Token string `toml:"token"` + LogDir string `toml:"log_dir"` + MirrorDir string `toml:"mirror_dir"` + Concurrent int `toml:"concurrent"` + Interval int `toml:"interval"` +} + +type mirrorConfig struct { + Name string `toml:"name"` + Provider ProviderEnum `toml:"provider"` + Upstream string `toml:"upstream"` + Interval int `toml:"interval"` + MirrorDir string `toml:"mirror_dir"` + LogDir string `toml:"log_dir"` + Env map[string]string `toml:"env"` + + Command string `toml:"command"` + UseIPv6 bool `toml:"use_ipv6"` + ExcludeFile string `toml:"exclude_file"` + Password string `toml:"password"` + Stage1Profile string `toml:"stage1_profile"` +} + +func loadConfig(cfgFile string) (*Config, error) { + if _, err := os.Stat(cfgFile); err != nil { + return nil, err + } + + cfg := new(Config) + if _, err := toml.DecodeFile(cfgFile, cfg); err != nil { + logger.Error(err.Error()) + return nil, err + } + return cfg, nil +} diff --git a/worker/config_test.go b/worker/config_test.go new file mode 100644 index 0000000..bcf43b9 --- /dev/null +++ b/worker/config_test.go @@ -0,0 +1,88 @@ +package worker + +import ( + "io/ioutil" + "os" + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestConfig(t *testing.T) { + var cfgBlob = ` +[global] +name = "test_worker" +token = "some_token" +log_dir = "/var/log/tunasync" +mirror_dir = "/data/mirrors" +concurrent = 10 +interval = 240 + +[[mirrors]] +name = "AOSP" +provider = "command" +upstream = "https://aosp.google.com/" +interval = 720 +mirror_dir = "/data/git/AOSP" + [mirrors.env] + REPO = "/usr/local/bin/aosp-repo" + +[[mirrors]] +name = "debian" +provider = "two-stage-rsync" +stage1_profile = "debian" +upstream = "rsync://ftp.debian.org/debian/" +use_ipv6 = true + + +[[mirrors]] +name = "fedora" +provider = "rsync" +upstream = "rsync://ftp.fedoraproject.org/fedora/" +use_ipv6 = true +exclude_file = "/etc/tunasync.d/fedora-exclude.txt" + ` + + Convey("When giving invalid file", t, func() { + cfg, err := loadConfig("/path/to/invalid/file") + So(err, ShouldNotBeNil) + So(cfg, ShouldBeNil) + }) + + Convey("Everything should work on valid config file", t, func() { + tmpfile, err := ioutil.TempFile("", "tunasync") + So(err, ShouldEqual, nil) + defer os.Remove(tmpfile.Name()) + + err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644) + So(err, ShouldEqual, nil) + defer tmpfile.Close() + + cfg, err := loadConfig(tmpfile.Name()) + So(err, ShouldBeNil) + So(cfg.Global.Name, ShouldEqual, "test_worker") + So(cfg.Global.Interval, ShouldEqual, 240) + So(cfg.Global.MirrorDir, ShouldEqual, "/data/mirrors") + + m := cfg.Mirrors[0] + So(m.Name, ShouldEqual, "AOSP") + So(m.MirrorDir, ShouldEqual, "/data/git/AOSP") + So(m.Provider, ShouldEqual, ProvCommand) + So(m.Interval, ShouldEqual, 720) + So(m.Env["REPO"], ShouldEqual, "/usr/local/bin/aosp-repo") + + m = cfg.Mirrors[1] + So(m.Name, ShouldEqual, "debian") + So(m.MirrorDir, ShouldEqual, "") + So(m.Provider, ShouldEqual, ProvTwoStageRsync) + + m = cfg.Mirrors[2] + So(m.Name, ShouldEqual, "fedora") + So(m.MirrorDir, ShouldEqual, "") + So(m.Provider, ShouldEqual, ProvRsync) + So(m.ExcludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt") + + So(len(cfg.Mirrors), ShouldEqual, 3) + + }) +} From f336fda7366dcc1aaa3e911014ea7de0466460b6 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 24 Apr 2016 21:52:15 +0800 Subject: [PATCH 24/66] feature(worker): mirrorConfig -> mirrorProvider --- worker/config.go | 8 ++++- worker/config_test.go | 9 +++-- worker/main.go | 75 +++++++++++++++++++++++++++++++++++++++- worker/rsync_provider.go | 13 ++++--- 4 files changed, 97 insertions(+), 8 deletions(-) diff --git a/worker/config.go b/worker/config.go index a2fe1bd..4c5e0bb 100644 --- a/worker/config.go +++ b/worker/config.go @@ -48,18 +48,24 @@ func (p *ProviderEnum) UnmarshalText(text []byte) error { type Config struct { Global globalConfig `toml:"global"` + Manager managerConfig `toml:"manager"` Mirrors []mirrorConfig `toml:"mirrors"` } type globalConfig struct { Name string `toml:"name"` - Token string `toml:"token"` LogDir string `toml:"log_dir"` MirrorDir string `toml:"mirror_dir"` Concurrent int `toml:"concurrent"` Interval int `toml:"interval"` } +type managerConfig struct { + APIBase string `toml:"api_base"` + CACert string `toml:"ca_cert"` + Token string `toml:"token"` +} + type mirrorConfig struct { Name string `toml:"name"` Provider ProviderEnum `toml:"provider"` diff --git a/worker/config_test.go b/worker/config_test.go index bcf43b9..c5ac4f3 100644 --- a/worker/config_test.go +++ b/worker/config_test.go @@ -12,12 +12,15 @@ func TestConfig(t *testing.T) { var cfgBlob = ` [global] name = "test_worker" -token = "some_token" -log_dir = "/var/log/tunasync" +log_dir = "/var/log/tunasync/{{.Name}}" mirror_dir = "/data/mirrors" concurrent = 10 interval = 240 +[manager] +api_base = "https://127.0.0.1:5000" +token = "some_token" + [[mirrors]] name = "AOSP" provider = "command" @@ -64,6 +67,8 @@ exclude_file = "/etc/tunasync.d/fedora-exclude.txt" So(cfg.Global.Interval, ShouldEqual, 240) So(cfg.Global.MirrorDir, ShouldEqual, "/data/mirrors") + So(cfg.Manager.APIBase, ShouldEqual, "https://127.0.0.1:5000") + m := cfg.Mirrors[0] So(m.Name, ShouldEqual, "AOSP") So(m.MirrorDir, ShouldEqual, "/data/git/AOSP") diff --git a/worker/main.go b/worker/main.go index b1d1453..df5ad9e 100644 --- a/worker/main.go +++ b/worker/main.go @@ -1,9 +1,82 @@ package worker -import "time" +import ( + "bytes" + "errors" + "html/template" + "path/filepath" + "time" +) // toplevel module for workers +func initProviders(c *Config) []mirrorProvider { + + formatLogDir := func(logDir string, m mirrorConfig) string { + tmpl, err := template.New("logDirTmpl-" + m.Name).Parse(logDir) + if err != nil { + panic(err) + } + var formatedLogDir bytes.Buffer + tmpl.Execute(&formatedLogDir, m) + return formatedLogDir.String() + } + + providers := []mirrorProvider{} + + for _, mirror := range c.Mirrors { + logDir := mirror.LogDir + mirrorDir := mirror.MirrorDir + if logDir == "" { + logDir = c.Global.LogDir + } + if mirrorDir == "" { + mirrorDir = c.Global.MirrorDir + } + logDir = formatLogDir(logDir, mirror) + switch mirror.Provider { + case ProvCommand: + pc := cmdConfig{ + name: mirror.Name, + upstreamURL: mirror.Upstream, + command: mirror.Command, + workingDir: filepath.Join(mirrorDir, mirror.Name), + logDir: logDir, + logFile: filepath.Join(logDir, "latest.log"), + interval: time.Duration(mirror.Interval) * time.Minute, + env: mirror.Env, + } + p, err := newCmdProvider(pc) + if err != nil { + panic(err) + } + providers = append(providers, p) + case ProvRsync: + rc := rsyncConfig{ + name: mirror.Name, + upstreamURL: mirror.Upstream, + password: mirror.Password, + excludeFile: mirror.ExcludeFile, + workingDir: filepath.Join(mirrorDir, mirror.Name), + logDir: logDir, + logFile: filepath.Join(logDir, "latest.log"), + useIPv6: mirror.UseIPv6, + interval: time.Duration(mirror.Interval) * time.Minute, + } + p, err := newRsyncProvider(rc) + if err != nil { + panic(err) + } + providers = append(providers, p) + default: + panic(errors.New("Invalid mirror provider")) + + } + + } + return providers +} + func main() { for { diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go index ba8dcf1..cf35a91 100644 --- a/worker/rsync_provider.go +++ b/worker/rsync_provider.go @@ -35,11 +35,16 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { } // TODO: implement this -func (p *rsyncProvider) Start() { - +func (p *rsyncProvider) Start() error { + return nil } // TODO: implement this -func (p *rsyncProvider) Terminate() { - +func (p *rsyncProvider) Terminate() error { + return nil +} + +// TODO: implement this +func (p *rsyncProvider) Wait() error { + return nil } From a6e8e9e2d9e208d5905f74dd102eba414bcf2335 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 24 Apr 2016 22:40:44 +0800 Subject: [PATCH 25/66] feature(worker): implemented rsync provider --- worker/cmd_provider.go | 29 +------------------------ worker/provider.go | 47 ++++++++++++++++++++++++++++++++++++++-- worker/rsync_provider.go | 43 +++++++++++++++++++++++++++--------- 3 files changed, 79 insertions(+), 40 deletions(-) diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go index 9bdaa09..61280c6 100644 --- a/worker/cmd_provider.go +++ b/worker/cmd_provider.go @@ -1,8 +1,6 @@ package worker import ( - "errors" - "os" "time" "github.com/anmitsu/go-shlex" @@ -20,8 +18,6 @@ type cmdProvider struct { baseProvider cmdConfig command []string - cmd *cmdJob - logFile *os.File } func newCmdProvider(c cmdConfig) (*cmdProvider, error) { @@ -59,32 +55,9 @@ func (p *cmdProvider) Start() error { env[k] = v } p.cmd = newCmdJob(p.command, p.WorkingDir(), env) - - logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { + if err := p.setLogFile(); err != nil { return err } - p.logFile = logFile - p.cmd.SetLogFile(logFile) return p.cmd.Start() } - -func (p *cmdProvider) Wait() error { - if p.logFile != nil { - defer p.logFile.Close() - } - return p.cmd.Wait() -} - -func (p *cmdProvider) Terminate() error { - logger.Debug("terminating provider: %s", p.Name()) - if p.cmd == nil { - return errors.New("provider command job not initialized") - } - if p.logFile != nil { - p.logFile.Close() - } - err := p.cmd.Terminate() - return err -} diff --git a/worker/provider.go b/worker/provider.go index 7aca550..700df5d 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -1,6 +1,10 @@ package worker -import "time" +import ( + "errors" + "os" + "time" +) // mirror provider is the wrapper of mirror jobs @@ -45,7 +49,11 @@ type baseProvider struct { ctx *Context name string interval time.Duration - hooks []jobHook + + cmd *cmdJob + logFile *os.File + + hooks []jobHook } func (p *baseProvider) Name() string { @@ -104,3 +112,38 @@ func (p *baseProvider) AddHook(hook jobHook) { func (p *baseProvider) Hooks() []jobHook { return p.hooks } + +func (p *baseProvider) setLogFile() error { + if p.LogFile() == "/dev/null" { + p.cmd.SetLogFile(nil) + return nil + } + + logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + logger.Error("Error opening logfile %s: %s", p.LogFile(), err.Error()) + return err + } + p.logFile = logFile + p.cmd.SetLogFile(logFile) + return nil +} + +func (p *baseProvider) Wait() error { + if p.logFile != nil { + defer p.logFile.Close() + } + return p.cmd.Wait() +} + +func (p *baseProvider) Terminate() error { + logger.Debug("terminating provider: %s", p.Name()) + if p.cmd == nil { + return errors.New("provider command job not initialized") + } + if p.logFile != nil { + p.logFile.Close() + } + err := p.cmd.Terminate() + return err +} diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go index cf35a91..5d25971 100644 --- a/worker/rsync_provider.go +++ b/worker/rsync_provider.go @@ -4,6 +4,7 @@ import "time" type rsyncConfig struct { name string + rsyncCmd string upstreamURL, password, excludeFile string workingDir, logDir, logFile string useIPv6 bool @@ -14,6 +15,7 @@ type rsyncConfig struct { type rsyncProvider struct { baseProvider rsyncConfig + options []string } func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { @@ -27,6 +29,25 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { rsyncConfig: c, } + if c.rsyncCmd == "" { + provider.rsyncCmd = "rsync" + } + + options := []string{ + "-aHvh", "--no-o", "--no-g", "--stats", + "--exclude", ".~tmp~/", + "--delete", "--delete-after", "--delay-updates", + "--safe-links", "--timeout=120", "--contimeout=120", + } + + if c.useIPv6 { + options = append(options, "-6") + } + + if c.excludeFile != "" { + options = append(options, "--exclude-from", c.excludeFile) + } + provider.ctx.Set(_WorkingDirKey, c.workingDir) provider.ctx.Set(_LogDirKey, c.logDir) provider.ctx.Set(_LogFileKey, c.logFile) @@ -34,17 +55,19 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { return provider, nil } -// TODO: implement this func (p *rsyncProvider) Start() error { - return nil -} + env := map[string]string{} + if p.password != "" { + env["RSYNC_PASSWORD"] = p.password + } + command := []string{p.rsyncCmd} + command = append(command, p.options...) + command = append(command, p.upstreamURL, p.WorkingDir()) -// TODO: implement this -func (p *rsyncProvider) Terminate() error { - return nil -} + p.cmd = newCmdJob(command, p.WorkingDir(), env) + if err := p.setLogFile(); err != nil { + return err + } -// TODO: implement this -func (p *rsyncProvider) Wait() error { - return nil + return p.cmd.Start() } From 9339fba074b4a8966bc49de082d80082bb402b11 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 24 Apr 2016 23:21:03 +0800 Subject: [PATCH 26/66] refactor(worker): use Run instead of Start and Wait --- worker/cmd_provider.go | 7 +++++++ worker/job.go | 11 ++-------- worker/provider.go | 45 ++++++++++++++++++++++++++++++---------- worker/provider_test.go | 15 ++++---------- worker/rsync_provider.go | 7 +++++++ worker/runner.go | 9 ++++---- 6 files changed, 58 insertions(+), 36 deletions(-) diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go index 61280c6..1e5a399 100644 --- a/worker/cmd_provider.go +++ b/worker/cmd_provider.go @@ -44,6 +44,13 @@ func newCmdProvider(c cmdConfig) (*cmdProvider, error) { return provider, nil } +func (p *cmdProvider) Run() error { + if err := p.Start(); err != nil { + return err + } + return p.Wait() +} + func (p *cmdProvider) Start() error { env := map[string]string{ "TUNASYNC_MIRROR_NAME": p.Name(), diff --git a/worker/job.go b/worker/job.go index 50d4c3d..186764b 100644 --- a/worker/job.go +++ b/worker/job.go @@ -103,18 +103,11 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err // start syncing managerChan <- jobMessage{tunasync.Syncing, m.Name(), ""} - err = provider.Start() - if err != nil { - logger.Error( - "failed to start syncing job for %s: %s", - m.Name(), err.Error(), - ) - return err - } + var syncErr error syncDone := make(chan error, 1) go func() { - err := provider.Wait() + err := provider.Run() if !stopASAP { syncDone <- err } diff --git a/worker/provider.go b/worker/provider.go index 700df5d..a27dbfe 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -3,6 +3,7 @@ package worker import ( "errors" "os" + "sync" "time" ) @@ -21,7 +22,8 @@ type mirrorProvider interface { // name Name() string - // TODO: implement Run, Terminate and Hooks + // run mirror job in background + Run() error // run mirror job in background Start() error // Wait job to finish @@ -46,6 +48,8 @@ type mirrorProvider interface { } type baseProvider struct { + sync.Mutex + ctx *Context name string interval time.Duration @@ -118,21 +122,35 @@ func (p *baseProvider) setLogFile() error { p.cmd.SetLogFile(nil) return nil } - - logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - logger.Error("Error opening logfile %s: %s", p.LogFile(), err.Error()) - return err + if p.logFile == nil { + logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + logger.Error("Error opening logfile %s: %s", p.LogFile(), err.Error()) + return err + } + p.logFile = logFile } - p.logFile = logFile - p.cmd.SetLogFile(logFile) + p.cmd.SetLogFile(p.logFile) return nil } +func (p *baseProvider) Run() error { + panic("Not Implemented") +} + +func (p *baseProvider) Start() error { + panic("Not Implemented") +} + func (p *baseProvider) Wait() error { - if p.logFile != nil { - defer p.logFile.Close() - } + defer func() { + p.Lock() + if p.logFile != nil { + p.logFile.Close() + p.logFile = nil + } + p.Unlock() + }() return p.cmd.Wait() } @@ -141,9 +159,14 @@ func (p *baseProvider) Terminate() error { if p.cmd == nil { return errors.New("provider command job not initialized") } + + p.Lock() if p.logFile != nil { p.logFile.Close() + p.logFile = nil } + p.Unlock() + err := p.cmd.Terminate() return err } diff --git a/worker/provider_test.go b/worker/provider_test.go index 79b5b03..13f550e 100644 --- a/worker/provider_test.go +++ b/worker/provider_test.go @@ -116,9 +116,7 @@ echo $AOSP_REPO_BIN So(err, ShouldBeNil) So(readedScriptContent, ShouldResemble, []byte(scriptContent)) - err = provider.Start() - So(err, ShouldBeNil) - err = provider.Wait() + err = provider.Run() So(err, ShouldBeNil) loggedContent, err := ioutil.ReadFile(provider.LogFile()) @@ -134,9 +132,7 @@ echo $AOSP_REPO_BIN So(err, ShouldBeNil) So(readedScriptContent, ShouldResemble, []byte(scriptContent)) - err = provider.Start() - So(err, ShouldBeNil) - err = provider.Wait() + err = provider.Run() So(err, ShouldNotBeNil) }) @@ -148,15 +144,12 @@ sleep 5 err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) So(err, ShouldBeNil) - err = provider.Start() - So(err, ShouldBeNil) - go func() { - err = provider.Wait() + err = provider.Run() ctx.So(err, ShouldNotBeNil) }() - time.Sleep(2) + time.Sleep(1 * time.Second) err = provider.Terminate() So(err, ShouldBeNil) diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go index 5d25971..83da9ec 100644 --- a/worker/rsync_provider.go +++ b/worker/rsync_provider.go @@ -55,6 +55,13 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { return provider, nil } +func (p *rsyncProvider) Run() error { + if err := p.Start(); err != nil { + return err + } + return p.Wait() +} + func (p *rsyncProvider) Start() error { env := map[string]string{} if p.password != "" { diff --git a/worker/runner.go b/worker/runner.go index ed6bc65..1e196a5 100644 --- a/worker/runner.go +++ b/worker/runner.go @@ -15,6 +15,8 @@ import ( // it's an alternative to python-sh or go-sh // TODO: cgroup excution +var errProcessNotStarted = errors.New("Process Not Started") + type cmdJob struct { cmd *exec.Cmd workingDir string @@ -62,11 +64,8 @@ func (c *cmdJob) SetLogFile(logFile *os.File) { } func (c *cmdJob) Terminate() error { - if c.cmd == nil { - return nil - } - if c.cmd.Process == nil { - return nil + if c.cmd == nil || c.cmd.Process == nil { + return errProcessNotStarted } err := unix.Kill(c.cmd.Process.Pid, syscall.SIGTERM) if err != nil { From 13161d77cfe91b1aab3c266f1401318ae3f40f67 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Mon, 25 Apr 2016 11:11:35 +0800 Subject: [PATCH 27/66] feature(worker): two-stage-rsync provider --- worker/cmd_provider.go | 6 +- worker/provider.go | 20 +++- worker/provider_test.go | 157 +++++++++++++++++++++++++++-- worker/rsync_provider.go | 17 +++- worker/runner.go | 2 +- worker/two_stage_rsync_provider.go | 136 +++++++++++++++++++++++++ 6 files changed, 323 insertions(+), 15 deletions(-) create mode 100644 worker/two_stage_rsync_provider.go diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go index 1e5a399..d6df6e7 100644 --- a/worker/cmd_provider.go +++ b/worker/cmd_provider.go @@ -66,5 +66,9 @@ func (p *cmdProvider) Start() error { return err } - return p.cmd.Start() + if err := p.cmd.Start(); err != nil { + return err + } + p.isRunning.Store(true) + return nil } diff --git a/worker/provider.go b/worker/provider.go index a27dbfe..143ccd8 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -1,9 +1,9 @@ package worker import ( - "errors" "os" "sync" + "sync/atomic" "time" ) @@ -31,6 +31,8 @@ type mirrorProvider interface { // terminate mirror job Terminate() error // job hooks + IsRunning() bool + Hooks() []jobHook Interval() time.Duration @@ -54,7 +56,9 @@ type baseProvider struct { name string interval time.Duration - cmd *cmdJob + cmd *cmdJob + isRunning atomic.Value + logFile *os.File hooks []jobHook @@ -142,9 +146,15 @@ func (p *baseProvider) Start() error { panic("Not Implemented") } +func (p *baseProvider) IsRunning() bool { + isRunning, _ := p.isRunning.Load().(bool) + return isRunning +} + func (p *baseProvider) Wait() error { defer func() { p.Lock() + p.isRunning.Store(false) if p.logFile != nil { p.logFile.Close() p.logFile = nil @@ -156,8 +166,8 @@ func (p *baseProvider) Wait() error { func (p *baseProvider) Terminate() error { logger.Debug("terminating provider: %s", p.Name()) - if p.cmd == nil { - return errors.New("provider command job not initialized") + if !p.IsRunning() { + return nil } p.Lock() @@ -168,5 +178,7 @@ func (p *baseProvider) Terminate() error { p.Unlock() err := p.cmd.Terminate() + p.isRunning.Store(false) + return err } diff --git a/worker/provider_test.go b/worker/provider_test.go index 13f550e..10fbceb 100644 --- a/worker/provider_test.go +++ b/worker/provider_test.go @@ -13,15 +13,21 @@ import ( func TestRsyncProvider(t *testing.T) { Convey("Rsync Provider should work", t, func() { + tmpDir, err := ioutil.TempDir("", "tunasync") + defer os.RemoveAll(tmpDir) + So(err, ShouldBeNil) + scriptFile := filepath.Join(tmpDir, "myrsync") + tmpFile := filepath.Join(tmpDir, "log_file") c := rsyncConfig{ name: "tuna", upstreamURL: "rsync://rsync.tuna.moe/tuna/", - workingDir: "/srv/mirror/production/tuna", - logDir: "/var/log/tunasync", - logFile: "tuna.log", + rsyncCmd: scriptFile, + workingDir: tmpDir, + logDir: tmpDir, + logFile: tmpFile, useIPv6: true, - interval: 600, + interval: 600 * time.Second, } provider, err := newRsyncProvider(c) @@ -61,6 +67,38 @@ func TestRsyncProvider(t *testing.T) { }) }) + Convey("Let's try a run", func() { + scriptContent := `#!/bin/bash +echo "syncing to $(pwd)" +echo $@ +sleep 1 +echo "Done" +exit 0 + ` + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + + expectedOutput := fmt.Sprintf( + "syncing to %s\n"+ + "%s\n"+ + "Done\n", + provider.WorkingDir(), + fmt.Sprintf( + "-aHvh --no-o --no-g --stats --exclude .~tmp~/ "+ + "--delete --delete-after --delay-updates --safe-links "+ + "--timeout=120 --contimeout=120 -6 %s %s", + provider.upstreamURL, provider.WorkingDir(), + ), + ) + + err = provider.Run() + So(err, ShouldBeNil) + loggedContent, err := ioutil.ReadFile(provider.LogFile()) + So(err, ShouldBeNil) + So(string(loggedContent), ShouldEqual, expectedOutput) + // fmt.Println(string(loggedContent)) + }) + }) } @@ -79,7 +117,7 @@ func TestCmdProvider(t *testing.T) { workingDir: tmpDir, logDir: tmpDir, logFile: tmpFile, - interval: 600, + interval: 600 * time.Second, env: map[string]string{ "AOSP_REPO_BIN": "/usr/local/bin/repo", }, @@ -102,7 +140,7 @@ echo $TUNASYNC_UPSTREAM_URL echo $TUNASYNC_LOG_FILE echo $AOSP_REPO_BIN ` - exceptedOutput := fmt.Sprintf( + expectedOutput := fmt.Sprintf( "%s\n%s\n%s\n%s\n%s\n", provider.WorkingDir(), provider.Name(), @@ -121,7 +159,7 @@ echo $AOSP_REPO_BIN loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) - So(string(loggedContent), ShouldEqual, exceptedOutput) + So(string(loggedContent), ShouldEqual, expectedOutput) }) Convey("If a command fails", func() { @@ -156,3 +194,108 @@ sleep 5 }) }) } + +func TestTwoStageRsyncProvider(t *testing.T) { + Convey("TwoStageRsync Provider should work", t, func(ctx C) { + tmpDir, err := ioutil.TempDir("", "tunasync") + defer os.RemoveAll(tmpDir) + So(err, ShouldBeNil) + scriptFile := filepath.Join(tmpDir, "myrsync") + tmpFile := filepath.Join(tmpDir, "log_file") + + c := twoStageRsyncConfig{ + name: "tuna-two-stage-rsync", + upstreamURL: "rsync://mirrors.tuna.moe/", + stage1Profile: "debian", + rsyncCmd: scriptFile, + workingDir: tmpDir, + logDir: tmpDir, + logFile: tmpFile, + useIPv6: true, + excludeFile: tmpFile, + } + + provider, err := newTwoStageRsyncProvider(c) + So(err, ShouldBeNil) + + So(provider.Name(), ShouldEqual, c.name) + So(provider.WorkingDir(), ShouldEqual, c.workingDir) + So(provider.LogDir(), ShouldEqual, c.logDir) + So(provider.LogFile(), ShouldEqual, c.logFile) + So(provider.Interval(), ShouldEqual, c.interval) + + Convey("Try a command", func(ctx C) { + scriptContent := `#!/bin/bash +echo "syncing to $(pwd)" +echo $@ +sleep 1 +echo "Done" +exit 0 + ` + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + + err = provider.Run() + So(err, ShouldBeNil) + + expectedOutput := fmt.Sprintf( + "syncing to %s\n"+ + "%s\n"+ + "Done\n"+ + "syncing to %s\n"+ + "%s\n"+ + "Done\n", + provider.WorkingDir(), + fmt.Sprintf( + "-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+ + "--timeout=120 --contimeout=120 --exclude dists/ -6 "+ + "--exclude-from %s %s %s", + provider.excludeFile, provider.upstreamURL, provider.WorkingDir(), + ), + provider.WorkingDir(), + fmt.Sprintf( + "-aHvh --no-o --no-g --stats --exclude .~tmp~/ "+ + "--delete --delete-after --delay-updates --safe-links "+ + "--timeout=120 --contimeout=120 -6 --exclude-from %s %s %s", + provider.excludeFile, provider.upstreamURL, provider.WorkingDir(), + ), + ) + + loggedContent, err := ioutil.ReadFile(provider.LogFile()) + So(err, ShouldBeNil) + So(string(loggedContent), ShouldEqual, expectedOutput) + // fmt.Println(string(loggedContent)) + + }) + Convey("Try terminating", func(ctx C) { + scriptContent := `#!/bin/bash +echo $@ +sleep 4 +exit 0 + ` + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + + go func() { + err = provider.Run() + ctx.So(err, ShouldNotBeNil) + }() + + time.Sleep(1 * time.Second) + err = provider.Terminate() + So(err, ShouldBeNil) + + expectedOutput := fmt.Sprintf( + "-aHvh --no-o --no-g --stats --exclude .~tmp~/ --safe-links "+ + "--timeout=120 --contimeout=120 --exclude dists/ -6 "+ + "--exclude-from %s %s %s\n", + provider.excludeFile, provider.upstreamURL, provider.WorkingDir(), + ) + + loggedContent, err := ioutil.ReadFile(provider.LogFile()) + So(err, ShouldBeNil) + So(string(loggedContent), ShouldEqual, expectedOutput) + // fmt.Println(string(loggedContent)) + }) + }) +} diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go index 83da9ec..34d8ddb 100644 --- a/worker/rsync_provider.go +++ b/worker/rsync_provider.go @@ -1,6 +1,10 @@ package worker -import "time" +import ( + "errors" + "strings" + "time" +) type rsyncConfig struct { name string @@ -20,6 +24,9 @@ type rsyncProvider struct { func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { // TODO: check config options + if !strings.HasSuffix(c.upstreamURL, "/") { + return nil, errors.New("rsync upstream URL should ends with /") + } provider := &rsyncProvider{ baseProvider: baseProvider{ name: c.name, @@ -47,6 +54,7 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { if c.excludeFile != "" { options = append(options, "--exclude-from", c.excludeFile) } + provider.options = options provider.ctx.Set(_WorkingDirKey, c.workingDir) provider.ctx.Set(_LogDirKey, c.logDir) @@ -63,6 +71,7 @@ func (p *rsyncProvider) Run() error { } func (p *rsyncProvider) Start() error { + env := map[string]string{} if p.password != "" { env["RSYNC_PASSWORD"] = p.password @@ -76,5 +85,9 @@ func (p *rsyncProvider) Start() error { return err } - return p.cmd.Start() + if err := p.cmd.Start(); err != nil { + return err + } + p.isRunning.Store(true) + return nil } diff --git a/worker/runner.go b/worker/runner.go index 1e196a5..aa8519f 100644 --- a/worker/runner.go +++ b/worker/runner.go @@ -54,7 +54,7 @@ func (c *cmdJob) Start() error { func (c *cmdJob) Wait() error { err := c.cmd.Wait() - c.finished <- empty{} + close(c.finished) return err } diff --git a/worker/two_stage_rsync_provider.go b/worker/two_stage_rsync_provider.go new file mode 100644 index 0000000..d5d7380 --- /dev/null +++ b/worker/two_stage_rsync_provider.go @@ -0,0 +1,136 @@ +package worker + +import ( + "errors" + "fmt" + "strings" + "time" +) + +type twoStageRsyncConfig struct { + name string + rsyncCmd string + stage1Profile string + upstreamURL, password, excludeFile string + workingDir, logDir, logFile string + useIPv6 bool + interval time.Duration +} + +// An RsyncProvider provides the implementation to rsync-based syncing jobs +type twoStageRsyncProvider struct { + baseProvider + twoStageRsyncConfig + stage1Options []string + stage2Options []string +} + +var rsyncStage1Profiles = map[string]([]string){ + "debian": []string{"dists/"}, + "debian-oldstyle": []string{ + "Packages*", "Sources*", "Release*", + "InRelease", "i18n/*", "ls-lR*", "dep11/*", + }, +} + +func newTwoStageRsyncProvider(c twoStageRsyncConfig) (*twoStageRsyncProvider, error) { + // TODO: check config options + if !strings.HasSuffix(c.upstreamURL, "/") { + return nil, errors.New("rsync upstream URL should ends with /") + } + + provider := &twoStageRsyncProvider{ + baseProvider: baseProvider{ + name: c.name, + ctx: NewContext(), + interval: c.interval, + }, + twoStageRsyncConfig: c, + stage1Options: []string{ + "-aHvh", "--no-o", "--no-g", "--stats", + "--exclude", ".~tmp~/", + "--safe-links", "--timeout=120", "--contimeout=120", + }, + stage2Options: []string{ + "-aHvh", "--no-o", "--no-g", "--stats", + "--exclude", ".~tmp~/", + "--delete", "--delete-after", "--delay-updates", + "--safe-links", "--timeout=120", "--contimeout=120", + }, + } + + if c.rsyncCmd == "" { + provider.rsyncCmd = "rsync" + } + + provider.ctx.Set(_WorkingDirKey, c.workingDir) + provider.ctx.Set(_LogDirKey, c.logDir) + provider.ctx.Set(_LogFileKey, c.logFile) + + return provider, nil +} + +func (p *twoStageRsyncProvider) Options(stage int) ([]string, error) { + var options []string + if stage == 1 { + options = append(options, p.stage1Options...) + stage1Excludes, ok := rsyncStage1Profiles[p.stage1Profile] + if !ok { + return nil, errors.New("Invalid Stage 1 Profile") + } + for _, exc := range stage1Excludes { + options = append(options, "--exclude", exc) + } + + } else if stage == 2 { + options = append(options, p.stage2Options...) + } else { + return []string{}, fmt.Errorf("Invalid stage: %d", stage) + } + + if p.useIPv6 { + options = append(options, "-6") + } + + if p.excludeFile != "" { + options = append(options, "--exclude-from", p.excludeFile) + } + + return options, nil +} + +func (p *twoStageRsyncProvider) Run() error { + + env := map[string]string{} + if p.password != "" { + env["RSYNC_PASSWORD"] = p.password + } + + stages := []int{1, 2} + for _, stage := range stages { + command := []string{p.rsyncCmd} + options, err := p.Options(stage) + if err != nil { + return err + } + command = append(command, options...) + command = append(command, p.upstreamURL, p.WorkingDir()) + + p.cmd = newCmdJob(command, p.WorkingDir(), env) + if err := p.setLogFile(); err != nil { + return err + } + + if err = p.cmd.Start(); err != nil { + return err + } + p.isRunning.Store(true) + + err = p.cmd.Wait() + p.isRunning.Store(false) + if err != nil { + return err + } + } + return nil +} From 731fba842f1ad31ef0c7b202f5955099c297fb95 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 10:44:21 +0800 Subject: [PATCH 28/66] feature(worker): job need to be started by jobStart signal --- worker/job.go | 6 +++++- worker/job_test.go | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/worker/job.go b/worker/job.go index 186764b..8a75efc 100644 --- a/worker/job.go +++ b/worker/job.go @@ -28,6 +28,7 @@ type jobMessage struct { type mirrorJob struct { provider mirrorProvider ctrlChan chan ctrlAction + stopped chan empty enabled bool } @@ -35,7 +36,7 @@ func newMirrorJob(provider mirrorProvider) *mirrorJob { return &mirrorJob{ provider: provider, ctrlChan: make(chan ctrlAction, 1), - enabled: true, + enabled: false, } } @@ -52,6 +53,9 @@ func (m *mirrorJob) Name() string { // TODO: message struct for managerChan func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) error { + m.stopped = make(chan empty) + defer close(m.stopped) + provider := m.provider // to make code shorter diff --git a/worker/job_test.go b/worker/job_test.go index 065d10e..15bc716 100644 --- a/worker/job_test.go +++ b/worker/job_test.go @@ -68,6 +68,15 @@ func TestMirrorJob(t *testing.T) { job := newMirrorJob(provider) go job.Run(managerChan, semaphore) + // job should not start if we don't start it + select { + case <-managerChan: + So(0, ShouldEqual, 1) // made this fail + case <-time.After(1 * time.Second): + So(0, ShouldEqual, 0) + } + + job.ctrlChan <- jobStart for i := 0; i < 2; i++ { msg := <-managerChan So(msg.status, ShouldEqual, PreSyncing) @@ -96,7 +105,7 @@ func TestMirrorJob(t *testing.T) { select { case <-managerChan: So(0, ShouldEqual, 1) // made this fail - case <-time.After(2 * time.Second): + case <-job.stopped: So(0, ShouldEqual, 0) } }) @@ -118,6 +127,7 @@ echo $TUNASYNC_WORKING_DIR Convey("If we kill it", func(ctx C) { go job.Run(managerChan, semaphore) + job.ctrlChan <- jobStart time.Sleep(1 * time.Second) msg := <-managerChan @@ -135,10 +145,12 @@ echo $TUNASYNC_WORKING_DIR So(err, ShouldBeNil) So(string(loggedContent), ShouldEqual, exceptedOutput) job.ctrlChan <- jobDisable + <-job.stopped }) Convey("If we don't kill it", func(ctx C) { go job.Run(managerChan, semaphore) + job.ctrlChan <- jobStart msg := <-managerChan So(msg.status, ShouldEqual, PreSyncing) @@ -156,6 +168,7 @@ echo $TUNASYNC_WORKING_DIR So(err, ShouldBeNil) So(string(loggedContent), ShouldEqual, exceptedOutput) job.ctrlChan <- jobDisable + <-job.stopped }) }) From 8b56fda1e9975fe49ef67f987d6259cb1777659a Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 10:50:51 +0800 Subject: [PATCH 29/66] feature(worker): added worker http server config --- worker/config.go | 9 +++++++++ worker/config_test.go | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/worker/config.go b/worker/config.go index 4c5e0bb..9c96aa1 100644 --- a/worker/config.go +++ b/worker/config.go @@ -49,6 +49,7 @@ func (p *ProviderEnum) UnmarshalText(text []byte) error { type Config struct { Global globalConfig `toml:"global"` Manager managerConfig `toml:"manager"` + Server serverConfig `toml:"server"` Mirrors []mirrorConfig `toml:"mirrors"` } @@ -66,6 +67,14 @@ type managerConfig struct { Token string `toml:"token"` } +type serverConfig struct { + Hostname string `toml:"hostname"` + Addr string `toml:"listen_addr"` + Port int `toml:"listen_port"` + SSLCert string `toml:"ssl_cert"` + SSLKey string `toml:"ssl_key"` +} + type mirrorConfig struct { Name string `toml:"name"` Provider ProviderEnum `toml:"provider"` diff --git a/worker/config_test.go b/worker/config_test.go index c5ac4f3..a4de3cb 100644 --- a/worker/config_test.go +++ b/worker/config_test.go @@ -21,6 +21,13 @@ interval = 240 api_base = "https://127.0.0.1:5000" token = "some_token" +[server] +hostname = "worker1.example.com" +listen_addr = "127.0.0.1" +listen_port = 6000 +ssl_cert = "/etc/tunasync.d/worker1.cert" +ssl_key = "/etc/tunasync.d/worker1.key" + [[mirrors]] name = "AOSP" provider = "command" @@ -68,6 +75,7 @@ exclude_file = "/etc/tunasync.d/fedora-exclude.txt" So(cfg.Global.MirrorDir, ShouldEqual, "/data/mirrors") So(cfg.Manager.APIBase, ShouldEqual, "https://127.0.0.1:5000") + So(cfg.Server.Hostname, ShouldEqual, "worker1.example.com") m := cfg.Mirrors[0] So(m.Name, ShouldEqual, "AOSP") From 23c3125cbfe1c464a21eba483ef6a6ded53f4d6f Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 11:14:18 +0800 Subject: [PATCH 30/66] tests(worker): added test for initProfile --- worker/config_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ worker/main.go | 18 ++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/worker/config_test.go b/worker/config_test.go index a4de3cb..ea461a2 100644 --- a/worker/config_test.go +++ b/worker/config_test.go @@ -96,6 +96,46 @@ exclude_file = "/etc/tunasync.d/fedora-exclude.txt" So(m.ExcludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt") So(len(cfg.Mirrors), ShouldEqual, 3) + }) + + Convey("Providers can be inited from a valid config file", t, func() { + tmpfile, err := ioutil.TempFile("", "tunasync") + So(err, ShouldEqual, nil) + defer os.Remove(tmpfile.Name()) + + err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644) + So(err, ShouldEqual, nil) + defer tmpfile.Close() + + cfg, err := loadConfig(tmpfile.Name()) + So(err, ShouldBeNil) + + providers := initProviders(cfg) + + p := providers[0] + So(p.Name(), ShouldEqual, "AOSP") + So(p.LogDir(), ShouldEqual, "/var/log/tunasync/AOSP") + So(p.LogFile(), ShouldEqual, "/var/log/tunasync/AOSP/latest.log") + _, ok := p.(*cmdProvider) + So(ok, ShouldBeTrue) + + p = providers[1] + So(p.Name(), ShouldEqual, "debian") + So(p.LogDir(), ShouldEqual, "/var/log/tunasync/debian") + So(p.LogFile(), ShouldEqual, "/var/log/tunasync/debian/latest.log") + r2p, ok := p.(*twoStageRsyncProvider) + So(ok, ShouldBeTrue) + So(r2p.stage1Profile, ShouldEqual, "debian") + So(r2p.WorkingDir(), ShouldEqual, "/data/mirrors/debian") + + p = providers[2] + So(p.Name(), ShouldEqual, "fedora") + So(p.LogDir(), ShouldEqual, "/var/log/tunasync/fedora") + So(p.LogFile(), ShouldEqual, "/var/log/tunasync/fedora/latest.log") + rp, ok := p.(*rsyncProvider) + So(ok, ShouldBeTrue) + So(rp.WorkingDir(), ShouldEqual, "/data/mirrors/fedora") + So(rp.excludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt") }) } diff --git a/worker/main.go b/worker/main.go index df5ad9e..a51f579 100644 --- a/worker/main.go +++ b/worker/main.go @@ -68,6 +68,24 @@ func initProviders(c *Config) []mirrorProvider { panic(err) } providers = append(providers, p) + case ProvTwoStageRsync: + rc := twoStageRsyncConfig{ + name: mirror.Name, + stage1Profile: mirror.Stage1Profile, + upstreamURL: mirror.Upstream, + password: mirror.Password, + excludeFile: mirror.ExcludeFile, + workingDir: filepath.Join(mirrorDir, mirror.Name), + logDir: logDir, + logFile: filepath.Join(logDir, "latest.log"), + useIPv6: mirror.UseIPv6, + interval: time.Duration(mirror.Interval) * time.Minute, + } + p, err := newTwoStageRsyncProvider(rc) + if err != nil { + panic(err) + } + providers = append(providers, p) default: panic(errors.New("Invalid mirror provider")) From 9afd47ddcb4b2dfb702d77ac129dfa81ce0fb562 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 14:58:43 +0800 Subject: [PATCH 31/66] feature(worker): LogLimiter hook --- worker/job.go | 12 ++++ worker/loglimit_hook.go | 108 +++++++++++++++++++++++++++++ worker/loglimit_test.go | 146 ++++++++++++++++++++++++++++++++++++++++ worker/provider.go | 1 + 4 files changed, 267 insertions(+) create mode 100644 worker/loglimit_hook.go create mode 100644 worker/loglimit_test.go diff --git a/worker/job.go b/worker/job.go index 8a75efc..d94cc3c 100644 --- a/worker/job.go +++ b/worker/job.go @@ -44,6 +44,18 @@ func (m *mirrorJob) Name() string { return m.provider.Name() } +func (m *mirrorJob) Stopped() bool { + if !m.enabled { + return true + } + select { + case <-m.stopped: + return true + default: + return false + } +} + // runMirrorJob is the goroutine where syncing job runs in // arguments: // provider: mirror provider object diff --git a/worker/loglimit_hook.go b/worker/loglimit_hook.go new file mode 100644 index 0000000..fdf55e3 --- /dev/null +++ b/worker/loglimit_hook.go @@ -0,0 +1,108 @@ +package worker + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// limit + +type logLimiter struct { + emptyHook + provider mirrorProvider +} + +func newLogLimiter(provider mirrorProvider) *logLimiter { + return &logLimiter{ + provider: provider, + } +} + +type fileSlice []os.FileInfo + +func (f fileSlice) Len() int { return len(f) } +func (f fileSlice) Swap(i, j int) { f[i], f[j] = f[j], f[i] } +func (f fileSlice) Less(i, j int) bool { return f[i].ModTime().Before(f[j].ModTime()) } + +func (l *logLimiter) preExec() error { + logger.Debug("executing log limitter for %s", l.provider.Name()) + + p := l.provider + if p.LogFile() == "/dev/null" { + return nil + } + + logDir := p.LogDir() + files, err := ioutil.ReadDir(logDir) + if err != nil { + if os.IsNotExist(err) { + os.MkdirAll(logDir, 0755) + } else { + return err + } + } + matchedFiles := []os.FileInfo{} + for _, f := range files { + if strings.HasPrefix(f.Name(), p.Name()) { + matchedFiles = append(matchedFiles, f) + } + } + + // sort the filelist in time order + // earlier modified files are sorted as larger + sort.Sort( + sort.Reverse( + fileSlice(matchedFiles), + ), + ) + // remove old files + if len(matchedFiles) > 9 { + for _, f := range matchedFiles[9:] { + // logger.Debug(f.Name()) + os.Remove(filepath.Join(logDir, f.Name())) + } + } + + logFile := filepath.Join( + logDir, + fmt.Sprintf( + "%s_%s.log", + p.Name(), + time.Now().Format("2006-01-02_15_04"), + ), + ) + + logLink := filepath.Join(logDir, "latest") + + if _, err = os.Stat(logLink); err == nil { + os.Remove(logLink) + } + os.Symlink(logFile, logLink) + + ctx := p.EnterContext() + ctx.Set(_LogFileKey, logFile) + return nil +} + +func (l *logLimiter) postSuccess() error { + l.provider.ExitContext() + return nil +} + +func (l *logLimiter) postFail() error { + logFile := l.provider.LogFile() + logFileFail := logFile + ".fail" + logDir := l.provider.LogDir() + logLink := filepath.Join(logDir, "latest") + os.Rename(logFile, logFileFail) + os.Remove(logLink) + os.Symlink(logFileFail, logLink) + + l.provider.ExitContext() + return nil +} diff --git a/worker/loglimit_test.go b/worker/loglimit_test.go new file mode 100644 index 0000000..74d87b6 --- /dev/null +++ b/worker/loglimit_test.go @@ -0,0 +1,146 @@ +package worker + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" + . "github.com/tuna/tunasync/internal" +) + +func TestLogLimiter(t *testing.T) { + Convey("LogLimiter should work", t, func(ctx C) { + tmpDir, err := ioutil.TempDir("", "tunasync") + tmpLogDir, err := ioutil.TempDir("", "tunasync-log") + defer os.RemoveAll(tmpDir) + defer os.RemoveAll(tmpLogDir) + So(err, ShouldBeNil) + scriptFile := filepath.Join(tmpDir, "cmd.sh") + + c := cmdConfig{ + name: "tuna-loglimit", + upstreamURL: "http://mirrors.tuna.moe/", + command: scriptFile, + workingDir: tmpDir, + logDir: tmpLogDir, + logFile: filepath.Join(tmpLogDir, "latest.log"), + interval: 600 * time.Second, + } + + provider, err := newCmdProvider(c) + So(err, ShouldBeNil) + limiter := newLogLimiter(provider) + provider.AddHook(limiter) + + Convey("If logs are created simply", func() { + for i := 0; i < 15; i++ { + fn := filepath.Join(tmpLogDir, fmt.Sprintf("%s-%d.log", provider.Name(), i)) + f, _ := os.Create(fn) + // time.Sleep(1 * time.Second) + f.Close() + } + + matches, _ := filepath.Glob(filepath.Join(tmpLogDir, "*.log")) + So(len(matches), ShouldEqual, 15) + + managerChan := make(chan jobMessage) + semaphore := make(chan empty, 1) + job := newMirrorJob(provider) + + scriptContent := `#!/bin/bash +echo $TUNASYNC_WORKING_DIR +echo $TUNASYNC_MIRROR_NAME +echo $TUNASYNC_UPSTREAM_URL +echo $TUNASYNC_LOG_FILE + ` + + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + + go job.Run(managerChan, semaphore) + job.ctrlChan <- jobStart + msg := <-managerChan + So(msg.status, ShouldEqual, PreSyncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Syncing) + logFile := provider.LogFile() + msg = <-managerChan + So(msg.status, ShouldEqual, Success) + + job.ctrlChan <- jobDisable + + So(logFile, ShouldNotEqual, provider.LogFile()) + + matches, _ = filepath.Glob(filepath.Join(tmpLogDir, "*.log")) + So(len(matches), ShouldEqual, 10) + + expectedOutput := fmt.Sprintf( + "%s\n%s\n%s\n%s\n", + provider.WorkingDir(), + provider.Name(), + provider.upstreamURL, + logFile, + ) + + loggedContent, err := ioutil.ReadFile(filepath.Join(provider.LogDir(), "latest")) + So(err, ShouldBeNil) + So(string(loggedContent), ShouldEqual, expectedOutput) + }) + + Convey("If job failed simply", func() { + managerChan := make(chan jobMessage) + semaphore := make(chan empty, 1) + job := newMirrorJob(provider) + + scriptContent := `#!/bin/bash +echo $TUNASYNC_WORKING_DIR +echo $TUNASYNC_MIRROR_NAME +echo $TUNASYNC_UPSTREAM_URL +echo $TUNASYNC_LOG_FILE +sleep 5 + ` + + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + + go job.Run(managerChan, semaphore) + job.ctrlChan <- jobStart + msg := <-managerChan + So(msg.status, ShouldEqual, PreSyncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Syncing) + logFile := provider.LogFile() + + time.Sleep(1 * time.Second) + job.ctrlChan <- jobStop + + msg = <-managerChan + So(msg.status, ShouldEqual, Failed) + + job.ctrlChan <- jobDisable + <-job.stopped + + So(logFile, ShouldNotEqual, provider.LogFile()) + + expectedOutput := fmt.Sprintf( + "%s\n%s\n%s\n%s\n", + provider.WorkingDir(), + provider.Name(), + provider.upstreamURL, + logFile, + ) + + loggedContent, err := ioutil.ReadFile(filepath.Join(provider.LogDir(), "latest")) + So(err, ShouldBeNil) + So(string(loggedContent), ShouldEqual, expectedOutput) + loggedContent, err = ioutil.ReadFile(logFile + ".fail") + So(err, ShouldBeNil) + So(string(loggedContent), ShouldEqual, expectedOutput) + }) + + }) +} diff --git a/worker/provider.go b/worker/provider.go index 143ccd8..7cd8b85 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -33,6 +33,7 @@ type mirrorProvider interface { // job hooks IsRunning() bool + AddHook(hook jobHook) Hooks() []jobHook Interval() time.Duration From 6062aa4b9d33ca0aeee6dc3b66f13c433601fd17 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 15:05:13 +0800 Subject: [PATCH 32/66] refactor(worker): rename provider.setLogFile to provider.prepareLogFile --- worker/cmd_provider.go | 2 +- worker/config.go | 15 --------------- worker/provider.go | 2 +- worker/rsync_provider.go | 2 +- worker/two_stage_rsync_provider.go | 2 +- 5 files changed, 4 insertions(+), 19 deletions(-) diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go index d6df6e7..38cdcac 100644 --- a/worker/cmd_provider.go +++ b/worker/cmd_provider.go @@ -62,7 +62,7 @@ func (p *cmdProvider) Start() error { env[k] = v } p.cmd = newCmdJob(p.command, p.WorkingDir(), env) - if err := p.setLogFile(); err != nil { + if err := p.prepareLogFile(); err != nil { return err } diff --git a/worker/config.go b/worker/config.go index 9c96aa1..f766ee1 100644 --- a/worker/config.go +++ b/worker/config.go @@ -15,21 +15,6 @@ const ( ProvCommand ) -func (p ProviderEnum) MarshalText() ([]byte, error) { - - switch p { - case ProvCommand: - return []byte("command"), nil - case ProvRsync: - return []byte("rsync"), nil - case ProvTwoStageRsync: - return []byte("two-stage-rsync"), nil - default: - return []byte{}, errors.New("Invalid ProviderEnum value") - } - -} - func (p *ProviderEnum) UnmarshalText(text []byte) error { s := string(text) switch s { diff --git a/worker/provider.go b/worker/provider.go index 7cd8b85..6aba4dc 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -122,7 +122,7 @@ func (p *baseProvider) Hooks() []jobHook { return p.hooks } -func (p *baseProvider) setLogFile() error { +func (p *baseProvider) prepareLogFile() error { if p.LogFile() == "/dev/null" { p.cmd.SetLogFile(nil) return nil diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go index 34d8ddb..96e88c7 100644 --- a/worker/rsync_provider.go +++ b/worker/rsync_provider.go @@ -81,7 +81,7 @@ func (p *rsyncProvider) Start() error { command = append(command, p.upstreamURL, p.WorkingDir()) p.cmd = newCmdJob(command, p.WorkingDir(), env) - if err := p.setLogFile(); err != nil { + if err := p.prepareLogFile(); err != nil { return err } diff --git a/worker/two_stage_rsync_provider.go b/worker/two_stage_rsync_provider.go index d5d7380..247aa0e 100644 --- a/worker/two_stage_rsync_provider.go +++ b/worker/two_stage_rsync_provider.go @@ -117,7 +117,7 @@ func (p *twoStageRsyncProvider) Run() error { command = append(command, p.upstreamURL, p.WorkingDir()) p.cmd = newCmdJob(command, p.WorkingDir(), env) - if err := p.setLogFile(); err != nil { + if err := p.prepareLogFile(); err != nil { return err } From bf31e168a2d7880293e2afbb9aea35e617a81bcc Mon Sep 17 00:00:00 2001 From: walkerning Date: Sun, 24 Apr 2016 22:33:42 +0800 Subject: [PATCH 33/66] feature(manager): implement manager server, to be tested --- internal/msg.go | 14 ++- manager/config.go | 5 + manager/db.go | 62 ++++++++++-- manager/server.go | 214 ++++++++++++++++++++++++++++++++++++++--- manager/server_test.go | 67 +++++++++++++ manager/status.go | 4 + 6 files changed, 341 insertions(+), 25 deletions(-) diff --git a/internal/msg.go b/internal/msg.go index d34116f..01647c1 100644 --- a/internal/msg.go +++ b/internal/msg.go @@ -15,9 +15,10 @@ type StatusUpdateMsg struct { ErrorMsg string `json:"error_msg"` } -// A WorkerInfoMsg is +// A WorkerInfoMsg is the information struct that describe +// a worker, and sent from the manager to clients. type WorkerInfoMsg struct { - Name string `json:"name"` + ID string `json:"id"` } type CmdVerb uint8 @@ -30,11 +31,16 @@ const ( CmdPing // ensure the goroutine is alive ) +// A WorkerCmd is the command message send from the +// manager to a worker type WorkerCmd struct { - Cmd CmdVerb `json:"cmd"` - Args []string `json:"args"` + Cmd CmdVerb `json:"cmd"` + MirrorID string `json:"mirror_id"` + Args []string `json:"args"` } +// A ClientCmd is the command message send from client +// to the manager type ClientCmd struct { Cmd CmdVerb `json:"cmd"` MirrorID string `json:"mirror_id"` diff --git a/manager/config.go b/manager/config.go index 9a422f7..155ee95 100644 --- a/manager/config.go +++ b/manager/config.go @@ -24,6 +24,7 @@ type ServerConfig struct { type FileConfig struct { StatusFile string `toml:"status_file"` DBFile string `toml:"db_file"` + DBType string `toml:"db_type"` // used to connect to worker CACert string `toml:"ca_cert"` } @@ -36,6 +37,7 @@ func loadConfig(cfgFile string, c *cli.Context) (*Config, error) { cfg.Debug = false cfg.Files.StatusFile = "/var/lib/tunasync/tunasync.json" cfg.Files.DBFile = "/var/lib/tunasync/tunasync.db" + cfg.Files.DBType = "bolt" if cfgFile != "" { if _, err := toml.DecodeFile(cfgFile, cfg); err != nil { @@ -60,6 +62,9 @@ func loadConfig(cfgFile string, c *cli.Context) (*Config, error) { if c.String("db-file") != "" { cfg.Files.DBFile = c.String("db-file") } + if c.String("db-type") != "" { + cfg.Files.DBFile = c.String("db-type") + } return cfg, nil } diff --git a/manager/db.go b/manager/db.go index 4cf3edd..1a1d3ed 100644 --- a/manager/db.go +++ b/manager/db.go @@ -1,13 +1,35 @@ package manager -import "github.com/boltdb/bolt" +import ( + "fmt" + "github.com/boltdb/bolt" +) type dbAdapter interface { - GetWorker(workerID string) - UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) - GetMirrorStatus(workerID, mirrorID string) - GetMirrorStatusList(workerID string) - Close() + ListWorkers() ([]worker, error) + GetWorker(workerID string) (worker, error) + CreateWorker(w worker) (worker, error) + UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) (mirrorStatus, error) + GetMirrorStatus(workerID, mirrorID string) (mirrorStatus, error) + ListMirrorStatus(workerID string) ([]mirrorStatus, error) + ListAllMirrorStatus() ([]mirrorStatus, error) + Close() error +} + +func makeDBAdapter(dbType string, dbFile string) (dbAdapter, error) { + if dbType == "bolt" { + innerDB, err := bolt.Open(dbFile, 0600, nil) + if err != nil { + return nil, err + } + db := boltAdapter{ + db: innerDB, + dbFile: dbFile, + } + return &db, nil + } + // unsupported db-type + return nil, fmt.Errorf("unsupported db-type: %s", dbType) } type boltAdapter struct { @@ -15,6 +37,34 @@ type boltAdapter struct { dbFile string } +func (b *boltAdapter) ListWorkers() ([]worker, error) { + return []worker{}, nil +} + +func (b *boltAdapter) GetWorker(workerID string) (worker, error) { + return worker{}, nil +} + +func (b *boltAdapter) CreateWorker(w worker) (worker, error) { + return worker{}, nil +} + +func (b *boltAdapter) UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) (mirrorStatus, error) { + return mirrorStatus{}, nil +} + +func (b *boltAdapter) GetMirrorStatus(workerID, mirrorID string) (mirrorStatus, error) { + return mirrorStatus{}, nil +} + +func (b *boltAdapter) ListMirrorStatus(workerID string) ([]mirrorStatus, error) { + return []mirrorStatus{}, nil +} + +func (b *boltAdapter) ListAllMirrorStatus() ([]mirrorStatus, error) { + return []mirrorStatus{}, nil +} + func (b *boltAdapter) Close() error { if b.db != nil { return b.db.Close() diff --git a/manager/server.go b/manager/server.go index 4d674bd..828e975 100644 --- a/manager/server.go +++ b/manager/server.go @@ -1,42 +1,226 @@ package manager import ( - "net/http" - + "fmt" "github.com/gin-gonic/gin" + . "github.com/tuna/tunasync/internal" + "net/http" + "sync" + "time" +) + +const ( + maxQueuedCmdNum = 3 + cmdPollTime = 10 * time.Second +) + +const ( + _errorKey = "error" + _infoKey = "message" ) type worker struct { // worker name - name string - // url to connect to worker - url string + id string // session token token string } -func makeHTTPServer(debug bool) *gin.Engine { +var ( + workerChannelMu sync.RWMutex + workerChannels = make(map[string]chan WorkerCmd) +) + +type managerServer struct { + *gin.Engine + adapter dbAdapter +} + +// listAllJobs repond with all jobs of specified workers +func (s *managerServer) listAllJobs(c *gin.Context) { + mirrorStatusList, err := s.adapter.ListAllMirrorStatus() + if err != nil { + err := fmt.Errorf("failed to list all mirror status: %s", + err.Error(), + ) + c.Error(err) + s.returnErrJSON(c, http.StatusInternalServerError, err) + return + } + c.JSON(http.StatusOK, mirrorStatusList) +} + +// listWrokers respond with informations of all the workers +func (s *managerServer) listWorkers(c *gin.Context) { + var workerInfos []WorkerInfoMsg + workers, err := s.adapter.ListWorkers() + if err != nil { + err := fmt.Errorf("failed to list workers: %s", + err.Error(), + ) + c.Error(err) + s.returnErrJSON(c, http.StatusInternalServerError, err) + return + } + for _, w := range workers { + workerInfos = append(workerInfos, + WorkerInfoMsg{w.id}) + } + c.JSON(http.StatusOK, workerInfos) +} + +// registerWorker register an newly-online worker +func (s *managerServer) registerWorker(c *gin.Context) { + var _worker worker + c.BindJSON(&_worker) + newWorker, err := s.adapter.CreateWorker(_worker) + if err != nil { + err := fmt.Errorf("failed to register worker: %s", + err.Error(), + ) + c.Error(err) + s.returnErrJSON(c, http.StatusInternalServerError, err) + return + } + // create workerCmd channel for this worker + workerChannelMu.Lock() + defer workerChannelMu.Unlock() + workerChannels[_worker.id] = make(chan WorkerCmd, maxQueuedCmdNum) + c.JSON(http.StatusOK, newWorker) +} + +// listJobsOfWorker respond with all the jobs of the specified worker +func (s *managerServer) listJobsOfWorker(c *gin.Context) { + workerID := c.Param("id") + mirrorStatusList, err := s.adapter.ListMirrorStatus(workerID) + if err != nil { + err := fmt.Errorf("failed to list jobs of worker %s: %s", + workerID, err.Error(), + ) + c.Error(err) + s.returnErrJSON(c, http.StatusInternalServerError, err) + return + } + c.JSON(http.StatusOK, mirrorStatusList) +} + +func (s *managerServer) returnErrJSON(c *gin.Context, code int, err error) { + c.JSON(code, gin.H{ + _errorKey: err.Error(), + }) +} + +func (s *managerServer) updateJobOfWorker(c *gin.Context) { + workerID := c.Param("id") + var status mirrorStatus + c.BindJSON(&status) + mirrorName := status.Name + newStatus, err := s.adapter.UpdateMirrorStatus(workerID, mirrorName, status) + if err != nil { + err := fmt.Errorf("failed to update job %s of worker %s: %s", + mirrorName, workerID, err.Error(), + ) + c.Error(err) + s.returnErrJSON(c, http.StatusInternalServerError, err) + return + } + c.JSON(http.StatusOK, newStatus) +} + +func (s *managerServer) handleClientCmd(c *gin.Context) { + workerChannelMu.RLock() + defer workerChannelMu.RUnlock() + var clientCmd ClientCmd + c.BindJSON(&clientCmd) + // TODO: decide which worker should do this mirror when WorkerID is null string + workerID := clientCmd.WorkerID + if workerID == "" { + // TODO: decide which worker should do this mirror when WorkerID is null string + logger.Error("handleClientCmd case workerID == \" \" not implemented yet") + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + workerChannel, ok := workerChannels[workerID] + if !ok { + err := fmt.Errorf("worker %s is not registered yet", workerID) + s.returnErrJSON(c, http.StatusBadRequest, err) + return + } + // parse client cmd into worker cmd + workerCmd := WorkerCmd{ + Cmd: clientCmd.Cmd, + MirrorID: clientCmd.MirrorID, + Args: clientCmd.Args, + } + select { + case workerChannel <- workerCmd: + // successfully insert command to channel + c.JSON(http.StatusOK, struct{}{}) + default: + // pending commands for that worker exceed + // the maxQueuedCmdNum threshold + err := fmt.Errorf("pending commands for worker %s exceed"+ + "the %d threshold, the command is dropped", + workerID, maxQueuedCmdNum) + c.Error(err) + s.returnErrJSON(c, http.StatusServiceUnavailable, err) + return + } +} + +func (s *managerServer) getCmdOfWorker(c *gin.Context) { + workerID := c.Param("id") + workerChannelMu.RLock() + defer workerChannelMu.RUnlock() + + workerChannel := workerChannels[workerID] + for { + select { + case _ = <-workerChannel: + // TODO: push new command to worker client + continue + case <-time.After(cmdPollTime): + // time limit exceeded, close the connection + break + } + } +} + +func (s *managerServer) setDBAdapter(adapter dbAdapter) { + s.adapter = adapter +} + +func makeHTTPServer(debug bool) *managerServer { + // create gin engine if !debug { gin.SetMode(gin.ReleaseMode) } - r := gin.Default() - r.GET("/ping", func(c *gin.Context) { + s := &managerServer{ + gin.Default(), + nil, + } + s.GET("/ping", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"msg": "pong"}) }) // list jobs, status page - r.GET("/jobs", func(c *gin.Context) {}) + s.GET("/jobs", s.listAllJobs) + + // list workers + s.GET("/workers", s.listWorkers) // worker online - r.POST("/workers/:name", func(c *gin.Context) {}) + s.POST("/workers/:id", s.registerWorker) + // get job list - r.GET("/workers/:name/jobs", func(c *gin.Context) {}) + s.GET("/workers/:id/jobs", s.listJobsOfWorker) // post job status - r.POST("/workers/:name/jobs/:job", func(c *gin.Context) {}) + s.POST("/workers/:id/jobs/:job", s.updateJobOfWorker) // worker command polling - r.GET("/workers/:name/cmd_stream", func(c *gin.Context) {}) + s.GET("/workers/:id/cmd_stream", s.getCmdOfWorker) // for tunasynctl to post commands - r.POST("/cmd/", func(c *gin.Context) {}) + s.POST("/cmd/", s.handleClientCmd) - return r + return s } diff --git a/manager/server_test.go b/manager/server_test.go index c6cb0e7..09ca72c 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -6,12 +6,79 @@ import ( "io/ioutil" "math/rand" "net/http" + "strings" "testing" "time" . "github.com/smartystreets/goconvey/convey" ) +type mockDBAdapter struct { + workerStore map[string]worker + statusStore map[string]mirrorStatus +} + +func (b *mockDBAdapter) ListWorkers() ([]worker, error) { + workers := make([]worker, len(b.workerStore)) + idx := 0 + for _, w := range b.workerStore { + workers[idx] = w + idx++ + } + return workers, nil +} + +func (b *mockDBAdapter) GetWorker(workerID string) (worker, error) { + w, ok := b.workerStore[workerID] + if !ok { + return worker{}, fmt.Errorf("inexist workerId") + } + return w, nil +} + +func (b *mockDBAdapter) CreateWorker(w worker) (worker, error) { + _, ok := b.workerStore[w.id] + if ok { + return worker{}, fmt.Errorf("duplicate worker name") + } + b.workerStore[w.id] = w + return w, nil +} + +func (b *mockDBAdapter) GetMirrorStatus(workerID, mirrorID string) (mirrorStatus, error) { + // TODO: need to check worker exist first + id := workerID + "/" + mirrorID + status, ok := b.statusStore[id] + if !ok { + return mirrorStatus{}, fmt.Errorf("no mirror %s exists in worker %s", mirrorID, workerID) + } + return status, nil +} + +func (b *mockDBAdapter) UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) (mirrorStatus, error) { + id := workerID + "/" + mirrorID + b.statusStore[id] = status + return status, nil +} + +func (b *mockDBAdapter) ListMirrorStatus(workerID string) ([]mirrorStatus, error) { + var mirrorStatusList []mirrorStatus + for k, v := range b.statusStore { + if wID := strings.Split(k, "/")[1]; wID == workerID { + mirrorStatusList = append(mirrorStatusList, v) + } + } + return mirrorStatusList, nil +} + +func (b *mockDBAdapter) ListAllMirrorStatus() ([]mirrorStatus, error) { + var mirrorStatusList []mirrorStatus + for _, v := range b.statusStore { + mirrorStatusList = append(mirrorStatusList, v) + } + return mirrorStatusList, nil +} + func TestHTTPServer(t *testing.T) { Convey("HTTP server should work", t, func() { s := makeHTTPServer(false) diff --git a/manager/status.go b/manager/status.go index c9e2c90..b360063 100644 --- a/manager/status.go +++ b/manager/status.go @@ -12,6 +12,8 @@ import ( type mirrorStatus struct { Name string + Worker string + IsMaster bool Status SyncStatus LastUpdate time.Time Upstream string @@ -21,6 +23,8 @@ type mirrorStatus struct { func (s mirrorStatus) MarshalJSON() ([]byte, error) { m := map[string]interface{}{ "name": s.Name, + "worker": s.Worker, + "is_master": s.IsMaster, "status": s.Status, "last_update": s.LastUpdate.Format("2006-01-02 15:04:05"), "last_update_ts": fmt.Sprintf("%d", s.LastUpdate.Unix()), From 4ea26921e74be4c657dcb68154ccb74414dd5205 Mon Sep 17 00:00:00 2001 From: walkerning Date: Mon, 25 Apr 2016 10:46:10 +0800 Subject: [PATCH 34/66] feature(manager): add fields in mirrorStatus --- manager/status.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/manager/status.go b/manager/status.go index b360063..4708163 100644 --- a/manager/status.go +++ b/manager/status.go @@ -49,6 +49,20 @@ func (s *mirrorStatus) UnmarshalJSON(v []byte) error { } else { return errors.New("key `name` does not exist in the json") } + if isMaster, ok := m["is_master"]; ok { + if s.IsMaster, ok = isMaster.(bool); !ok { + return errors.New("is_master should be a string") + } + } else { + return errors.New("key `is_master` does not exist in the json") + } + if _worker, ok := m["worker"]; ok { + if s.Worker, ok = _worker.(string); !ok { + return errors.New("worker should be a string") + } + } else { + return errors.New("key `worker` does not exist in the json") + } if upstream, ok := m["upstream"]; ok { if s.Upstream, ok = upstream.(string); !ok { return errors.New("upstream should be a string") From 02bb8c16abc05535d798a887bc2b7b0b3ce26bba Mon Sep 17 00:00:00 2001 From: walkerning Date: Mon, 25 Apr 2016 10:47:13 +0800 Subject: [PATCH 35/66] feature(manager): add contextErrorLogger middleware --- manager/middleware.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 manager/middleware.go diff --git a/manager/middleware.go b/manager/middleware.go new file mode 100644 index 0000000..df00426 --- /dev/null +++ b/manager/middleware.go @@ -0,0 +1,16 @@ +package manager + +import ( + "github.com/gin-gonic/gin" +) + +func contextErrorLogger(c *gin.Context) { + errs := c.Errors.ByType(gin.ErrorTypeAny) + if len(errs) > 0 { + for _, err := range errs { + logger.Error(`"in request "%s %s: %s"`, c.Request.Method, c.Request.URL.Path, err.Error()) + } + } + // pass on to the next middleware in chain + c.Next() +} From 401b6a694e980963ed73fd8c1d274bc910c0ae01 Mon Sep 17 00:00:00 2001 From: walkerning Date: Mon, 25 Apr 2016 10:47:29 +0800 Subject: [PATCH 36/66] tests(manager): add tests for server.go, validate workerID in middleware --- manager/middleware.go | 17 ++++ manager/server.go | 26 +++--- manager/server_test.go | 189 ++++++++++++++++++++++++++++++++++------- 3 files changed, 190 insertions(+), 42 deletions(-) diff --git a/manager/middleware.go b/manager/middleware.go index df00426..f620261 100644 --- a/manager/middleware.go +++ b/manager/middleware.go @@ -1,6 +1,9 @@ package manager import ( + "fmt" + "net/http" + "github.com/gin-gonic/gin" ) @@ -14,3 +17,17 @@ func contextErrorLogger(c *gin.Context) { // pass on to the next middleware in chain c.Next() } + +func (s *managerServer) workerIDValidator(c *gin.Context) { + workerID := c.Param("id") + _, err := s.adapter.GetWorker(workerID) + if err != nil { + // no worker named `workerID` exists + err := fmt.Errorf("invalid workerID %s", workerID) + s.returnErrJSON(c, http.StatusBadRequest, err) + c.Abort() + return + } + // pass on to the next middleware in chain + c.Next() +} diff --git a/manager/server.go b/manager/server.go index 828e975..70a2b00 100644 --- a/manager/server.go +++ b/manager/server.go @@ -20,10 +20,8 @@ const ( ) type worker struct { - // worker name - id string - // session token - token string + ID string `json:"id"` // worker name + Token string `json:"token"` // session token } var ( @@ -64,7 +62,7 @@ func (s *managerServer) listWorkers(c *gin.Context) { } for _, w := range workers { workerInfos = append(workerInfos, - WorkerInfoMsg{w.id}) + WorkerInfoMsg{w.ID}) } c.JSON(http.StatusOK, workerInfos) } @@ -85,7 +83,7 @@ func (s *managerServer) registerWorker(c *gin.Context) { // create workerCmd channel for this worker workerChannelMu.Lock() defer workerChannelMu.Unlock() - workerChannels[_worker.id] = make(chan WorkerCmd, maxQueuedCmdNum) + workerChannels[_worker.ID] = make(chan WorkerCmd, maxQueuedCmdNum) c.JSON(http.StatusOK, newWorker) } @@ -200,8 +198,12 @@ func makeHTTPServer(debug bool) *managerServer { gin.Default(), nil, } + + // common log middleware + s.Use(contextErrorLogger) + s.GET("/ping", func(c *gin.Context) { - c.JSON(http.StatusOK, gin.H{"msg": "pong"}) + c.JSON(http.StatusOK, gin.H{_infoKey: "pong"}) }) // list jobs, status page s.GET("/jobs", s.listAllJobs) @@ -209,15 +211,17 @@ func makeHTTPServer(debug bool) *managerServer { // list workers s.GET("/workers", s.listWorkers) // worker online - s.POST("/workers/:id", s.registerWorker) + s.POST("/workers", s.registerWorker) + // workerID should be valid in this route group + workerValidateGroup := s.Group("/workers", s.workerIDValidator) // get job list - s.GET("/workers/:id/jobs", s.listJobsOfWorker) + workerValidateGroup.GET(":id/jobs", s.listJobsOfWorker) // post job status - s.POST("/workers/:id/jobs/:job", s.updateJobOfWorker) + workerValidateGroup.POST(":id/jobs/:job", s.updateJobOfWorker) // worker command polling - s.GET("/workers/:id/cmd_stream", s.getCmdOfWorker) + workerValidateGroup.GET(":id/cmd_stream", s.getCmdOfWorker) // for tunasynctl to post commands s.POST("/cmd/", s.handleClientCmd) diff --git a/manager/server_test.go b/manager/server_test.go index 09ca72c..cfc229b 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -1,6 +1,7 @@ package manager import ( + "bytes" "encoding/json" "fmt" "io/ioutil" @@ -11,8 +12,146 @@ import ( "time" . "github.com/smartystreets/goconvey/convey" + . "github.com/tuna/tunasync/internal" ) +const ( + _magicBadWorkerID = "magic_bad_worker_id" +) + +func postJSON(url string, obj interface{}) (*http.Response, error) { + b := new(bytes.Buffer) + json.NewEncoder(b).Encode(obj) + return http.Post(url, "application/json; charset=utf-8", b) +} + +func TestHTTPServer(t *testing.T) { + Convey("HTTP server should work", t, func() { + InitLogger(true, true, false) + s := makeHTTPServer(false) + So(s, ShouldNotBeNil) + s.setDBAdapter(&mockDBAdapter{ + workerStore: map[string]worker{ + _magicBadWorkerID: worker{ + ID: _magicBadWorkerID, + }}, + statusStore: make(map[string]mirrorStatus), + }) + port := rand.Intn(10000) + 20000 + baseURL := fmt.Sprintf("http://127.0.0.1:%d", port) + go func() { + s.Run(fmt.Sprintf("127.0.0.1:%d", port)) + }() + time.Sleep(50 * time.Microsecond) + resp, err := http.Get(baseURL + "/ping") + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + So(resp.Header.Get("Content-Type"), ShouldEqual, "application/json; charset=utf-8") + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + So(err, ShouldBeNil) + var p map[string]string + err = json.Unmarshal(body, &p) + So(err, ShouldBeNil) + So(p[_infoKey], ShouldEqual, "pong") + + Convey("when database fail", func() { + resp, err := http.Get(fmt.Sprintf("%s/workers/%s/jobs", baseURL, _magicBadWorkerID)) + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError) + defer resp.Body.Close() + var msg map[string]string + err = json.NewDecoder(resp.Body).Decode(&msg) + So(err, ShouldBeNil) + So(msg[_errorKey], ShouldEqual, fmt.Sprintf("failed to list jobs of worker %s: %s", _magicBadWorkerID, "database fail")) + }) + + Convey("when register a worker", func() { + w := worker{ + ID: "test_worker1", + } + resp, err := postJSON(baseURL+"/workers", w) + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + + Convey("list all workers", func() { + So(err, ShouldBeNil) + resp, err := http.Get(baseURL + "/workers") + So(err, ShouldBeNil) + defer resp.Body.Close() + var actualResponseObj []WorkerInfoMsg + err = json.NewDecoder(resp.Body).Decode(&actualResponseObj) + So(err, ShouldBeNil) + So(len(actualResponseObj), ShouldEqual, 2) + }) + + Convey("update mirror status of a existed worker", func() { + status := mirrorStatus{ + Name: "arch-sync1", + Worker: "test_worker1", + IsMaster: true, + Status: Success, + LastUpdate: time.Now(), + Upstream: "mirrors.tuna.tsinghua.edu.cn", + Size: "3GB", + } + resp, err := postJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", baseURL, status.Worker, status.Name), status) + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + + Convey("list mirror status of an existed worker", func() { + + expectedResponse, err := json.Marshal([]mirrorStatus{status}) + So(err, ShouldBeNil) + resp, err := http.Get(baseURL + "/workers/test_worker1/jobs") + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + // err = json.NewDecoder(resp.Body).Decode(&mirrorStatusList) + body, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + So(err, ShouldBeNil) + So(strings.TrimSpace(string(body)), ShouldEqual, string(expectedResponse)) + }) + + Convey("list all job status of all workers", func() { + expectedResponse, err := json.Marshal([]mirrorStatus{status}) + So(err, ShouldBeNil) + resp, err := http.Get(baseURL + "/jobs") + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + body, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + So(err, ShouldBeNil) + So(strings.TrimSpace(string(body)), ShouldEqual, string(expectedResponse)) + + }) + }) + + Convey("update mirror status of an inexisted worker", func() { + invalidWorker := "test_worker2" + status := mirrorStatus{ + Name: "arch-sync2", + Worker: invalidWorker, + IsMaster: true, + Status: Success, + LastUpdate: time.Now(), + Upstream: "mirrors.tuna.tsinghua.edu.cn", + Size: "4GB", + } + resp, err := postJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", + baseURL, status.Worker, status.Name), status) + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusBadRequest) + defer resp.Body.Close() + var msg map[string]string + err = json.NewDecoder(resp.Body).Decode(&msg) + So(err, ShouldBeNil) + So(msg[_errorKey], ShouldEqual, "invalid workerID "+invalidWorker) + }) + }) + }) +} + type mockDBAdapter struct { workerStore map[string]worker statusStore map[string]mirrorStatus @@ -31,23 +170,22 @@ func (b *mockDBAdapter) ListWorkers() ([]worker, error) { func (b *mockDBAdapter) GetWorker(workerID string) (worker, error) { w, ok := b.workerStore[workerID] if !ok { - return worker{}, fmt.Errorf("inexist workerId") + return worker{}, fmt.Errorf("invalid workerId") } return w, nil } func (b *mockDBAdapter) CreateWorker(w worker) (worker, error) { - _, ok := b.workerStore[w.id] - if ok { - return worker{}, fmt.Errorf("duplicate worker name") - } - b.workerStore[w.id] = w + // _, ok := b.workerStore[w.ID] + // if ok { + // return worker{}, fmt.Errorf("duplicate worker name") + // } + b.workerStore[w.ID] = w return w, nil } func (b *mockDBAdapter) GetMirrorStatus(workerID, mirrorID string) (mirrorStatus, error) { - // TODO: need to check worker exist first - id := workerID + "/" + mirrorID + id := mirrorID + "/" + workerID status, ok := b.statusStore[id] if !ok { return mirrorStatus{}, fmt.Errorf("no mirror %s exists in worker %s", mirrorID, workerID) @@ -56,13 +194,22 @@ func (b *mockDBAdapter) GetMirrorStatus(workerID, mirrorID string) (mirrorStatus } func (b *mockDBAdapter) UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) (mirrorStatus, error) { - id := workerID + "/" + mirrorID + // if _, ok := b.workerStore[workerID]; !ok { + // // unregistered worker + // return mirrorStatus{}, fmt.Errorf("invalid workerID %s", workerID) + // } + + id := mirrorID + "/" + workerID b.statusStore[id] = status return status, nil } func (b *mockDBAdapter) ListMirrorStatus(workerID string) ([]mirrorStatus, error) { var mirrorStatusList []mirrorStatus + // simulating a database fail + if workerID == _magicBadWorkerID { + return []mirrorStatus{}, fmt.Errorf("database fail") + } for k, v := range b.statusStore { if wID := strings.Split(k, "/")[1]; wID == workerID { mirrorStatusList = append(mirrorStatusList, v) @@ -79,26 +226,6 @@ func (b *mockDBAdapter) ListAllMirrorStatus() ([]mirrorStatus, error) { return mirrorStatusList, nil } -func TestHTTPServer(t *testing.T) { - Convey("HTTP server should work", t, func() { - s := makeHTTPServer(false) - So(s, ShouldNotBeNil) - port := rand.Intn(10000) + 20000 - go func() { - s.Run(fmt.Sprintf("127.0.0.1:%d", port)) - }() - time.Sleep(50 * time.Microsecond) - resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/ping", port)) - So(err, ShouldBeNil) - So(resp.StatusCode, ShouldEqual, http.StatusOK) - So(resp.Header.Get("Content-Type"), ShouldEqual, "application/json; charset=utf-8") - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - So(err, ShouldBeNil) - var p map[string]string - err = json.Unmarshal(body, &p) - So(err, ShouldBeNil) - So(p["msg"], ShouldEqual, "pong") - }) - +func (b *mockDBAdapter) Close() error { + return nil } From 00eddc306616eb9795792172dd285afd3c9b8376 Mon Sep 17 00:00:00 2001 From: walkerning Date: Mon, 25 Apr 2016 19:05:04 +0800 Subject: [PATCH 37/66] feature(manager): add LastOnline feild to worker struct --- internal/msg.go | 3 ++- manager/server.go | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/msg.go b/internal/msg.go index 01647c1..b1478f2 100644 --- a/internal/msg.go +++ b/internal/msg.go @@ -18,7 +18,8 @@ type StatusUpdateMsg struct { // A WorkerInfoMsg is the information struct that describe // a worker, and sent from the manager to clients. type WorkerInfoMsg struct { - ID string `json:"id"` + ID string `json:"id"` + LastOnline time.Time `json:"last_online"` } type CmdVerb uint8 diff --git a/manager/server.go b/manager/server.go index 70a2b00..38aa77f 100644 --- a/manager/server.go +++ b/manager/server.go @@ -2,11 +2,13 @@ package manager import ( "fmt" - "github.com/gin-gonic/gin" - . "github.com/tuna/tunasync/internal" "net/http" "sync" "time" + + "github.com/gin-gonic/gin" + + . "github.com/tuna/tunasync/internal" ) const ( @@ -20,8 +22,9 @@ const ( ) type worker struct { - ID string `json:"id"` // worker name - Token string `json:"token"` // session token + ID string `json:"id"` // worker name + Token string `json:"token"` // session token + LastOnline time.Time `json:"last_online"` // last seen } var ( @@ -62,7 +65,7 @@ func (s *managerServer) listWorkers(c *gin.Context) { } for _, w := range workers { workerInfos = append(workerInfos, - WorkerInfoMsg{w.ID}) + WorkerInfoMsg{w.ID, w.LastOnline}) } c.JSON(http.StatusOK, workerInfos) } From a11fbe2c58885ba6fd27b0a4f00c29ec5156044c Mon Sep 17 00:00:00 2001 From: walkerning Date: Mon, 25 Apr 2016 19:05:27 +0800 Subject: [PATCH 38/66] feature(manager): implement db.go and tests --- manager/db.go | 129 ++++++++++++++++++++++++++++++++++++----- manager/db_test.go | 117 +++++++++++++++++++++++++++++++++++++ manager/server_test.go | 4 ++ 3 files changed, 237 insertions(+), 13 deletions(-) create mode 100644 manager/db_test.go diff --git a/manager/db.go b/manager/db.go index 1a1d3ed..36d9ff6 100644 --- a/manager/db.go +++ b/manager/db.go @@ -1,11 +1,15 @@ package manager import ( + "encoding/json" "fmt" + "strings" + "github.com/boltdb/bolt" ) type dbAdapter interface { + Init() error ListWorkers() ([]worker, error) GetWorker(workerID string) (worker, error) CreateWorker(w worker) (worker, error) @@ -26,43 +30,142 @@ func makeDBAdapter(dbType string, dbFile string) (dbAdapter, error) { db: innerDB, dbFile: dbFile, } - return &db, nil + err = db.Init() + return &db, err } // unsupported db-type return nil, fmt.Errorf("unsupported db-type: %s", dbType) } +const ( + _workerBucketKey = "workers" + _statusBucketKey = "mirror_status" +) + type boltAdapter struct { db *bolt.DB dbFile string } -func (b *boltAdapter) ListWorkers() ([]worker, error) { - return []worker{}, nil +func (b *boltAdapter) Init() (err error) { + return b.db.Update(func(tx *bolt.Tx) error { + _, err = tx.CreateBucketIfNotExists([]byte(_workerBucketKey)) + if err != nil { + return fmt.Errorf("create bucket %s error: %s", _workerBucketKey, err.Error()) + } + _, err = tx.CreateBucketIfNotExists([]byte(_statusBucketKey)) + if err != nil { + return fmt.Errorf("create bucket %s error: %s", _statusBucketKey, err.Error()) + } + return nil + }) } -func (b *boltAdapter) GetWorker(workerID string) (worker, error) { - return worker{}, nil +func (b *boltAdapter) ListWorkers() (ws []worker, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(_workerBucketKey)) + c := bucket.Cursor() + var w worker + for k, v := c.First(); k != nil; k, v = c.Next() { + jsonErr := json.Unmarshal(v, &w) + if jsonErr != nil { + err = fmt.Errorf("%s; %s", err.Error(), jsonErr) + continue + } + ws = append(ws, w) + } + return err + }) + return +} + +func (b *boltAdapter) GetWorker(workerID string) (w worker, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(_workerBucketKey)) + v := bucket.Get([]byte(workerID)) + if v == nil { + return fmt.Errorf("invalid workerID %s", workerID) + } + err := json.Unmarshal(v, &w) + return err + }) + return } func (b *boltAdapter) CreateWorker(w worker) (worker, error) { - return worker{}, nil + err := b.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(_workerBucketKey)) + v, err := json.Marshal(w) + if err != nil { + return err + } + err = bucket.Put([]byte(w.ID), v) + return err + }) + return w, err } func (b *boltAdapter) UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) (mirrorStatus, error) { - return mirrorStatus{}, nil + id := mirrorID + "/" + workerID + err := b.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(_statusBucketKey)) + v, err := json.Marshal(status) + err = bucket.Put([]byte(id), v) + return err + }) + return status, err } -func (b *boltAdapter) GetMirrorStatus(workerID, mirrorID string) (mirrorStatus, error) { - return mirrorStatus{}, nil +func (b *boltAdapter) GetMirrorStatus(workerID, mirrorID string) (m mirrorStatus, err error) { + id := mirrorID + "/" + workerID + err = b.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(_statusBucketKey)) + v := bucket.Get([]byte(id)) + if v == nil { + return fmt.Errorf("no mirror %s exists in worker %s", mirrorID, workerID) + } + err := json.Unmarshal(v, &m) + return err + }) + return } -func (b *boltAdapter) ListMirrorStatus(workerID string) ([]mirrorStatus, error) { - return []mirrorStatus{}, nil +func (b *boltAdapter) ListMirrorStatus(workerID string) (ms []mirrorStatus, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(_statusBucketKey)) + c := bucket.Cursor() + var m mirrorStatus + for k, v := c.First(); k != nil; k, v = c.Next() { + if wID := strings.Split(string(k), "/")[1]; wID == workerID { + jsonErr := json.Unmarshal(v, &m) + if jsonErr != nil { + err = fmt.Errorf("%s; %s", err.Error(), jsonErr) + continue + } + ms = append(ms, m) + } + } + return err + }) + return } -func (b *boltAdapter) ListAllMirrorStatus() ([]mirrorStatus, error) { - return []mirrorStatus{}, nil +func (b *boltAdapter) ListAllMirrorStatus() (ms []mirrorStatus, err error) { + err = b.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(_statusBucketKey)) + c := bucket.Cursor() + var m mirrorStatus + for k, v := c.First(); k != nil; k, v = c.Next() { + jsonErr := json.Unmarshal(v, &m) + if jsonErr != nil { + err = fmt.Errorf("%s; %s", err.Error(), jsonErr) + continue + } + ms = append(ms, m) + } + return err + }) + return } func (b *boltAdapter) Close() error { diff --git a/manager/db_test.go b/manager/db_test.go new file mode 100644 index 0000000..e95dfe8 --- /dev/null +++ b/manager/db_test.go @@ -0,0 +1,117 @@ +package manager + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" + . "github.com/tuna/tunasync/internal" +) + +func TestBoltAdapter(t *testing.T) { + Convey("boltAdapter should work", t, func() { + tmpDir, err := ioutil.TempDir("", "tunasync") + defer os.RemoveAll(tmpDir) + So(err, ShouldBeNil) + + dbType, dbFile := "bolt", filepath.Join(tmpDir, "bolt.db") + boltDB, err := makeDBAdapter(dbType, dbFile) + So(err, ShouldBeNil) + + defer func() { + // close boltDB + err := boltDB.Close() + So(err, ShouldBeNil) + }() + + testWorkerIDs := []string{"test_worker1", "test_worker2"} + Convey("create worker", func() { + for _, id := range testWorkerIDs { + w := worker{ + ID: id, + Token: "token_" + id, + LastOnline: time.Now(), + } + w, err = boltDB.CreateWorker(w) + So(err, ShouldBeNil) + } + + Convey("get exists worker", func() { + _, err := boltDB.GetWorker(testWorkerIDs[0]) + So(err, ShouldBeNil) + }) + + Convey("list exist worker", func() { + ws, err := boltDB.ListWorkers() + So(err, ShouldBeNil) + So(len(ws), ShouldEqual, 2) + }) + + Convey("get inexist worker", func() { + _, err := boltDB.GetWorker("invalid workerID") + So(err, ShouldNotBeNil) + }) + }) + + Convey("update mirror status", func() { + status1 := mirrorStatus{ + Name: "arch-sync1", + Worker: testWorkerIDs[0], + IsMaster: true, + Status: Success, + LastUpdate: time.Now(), + Upstream: "mirrors.tuna.tsinghua.edu.cn", + Size: "3GB", + } + status2 := mirrorStatus{ + Name: "arch-sync2", + Worker: testWorkerIDs[1], + IsMaster: true, + Status: Success, + LastUpdate: time.Now(), + Upstream: "mirrors.tuna.tsinghua.edu.cn", + Size: "4GB", + } + + _, err := boltDB.UpdateMirrorStatus(status1.Worker, status1.Name, status1) + _, err = boltDB.UpdateMirrorStatus(status2.Worker, status2.Name, status2) + So(err, ShouldBeNil) + + Convey("get mirror status", func() { + m, err := boltDB.GetMirrorStatus(testWorkerIDs[0], status1.Name) + So(err, ShouldBeNil) + expectedJSON, err := json.Marshal(status1) + So(err, ShouldBeNil) + actualJSON, err := json.Marshal(m) + So(err, ShouldBeNil) + So(string(actualJSON), ShouldEqual, string(expectedJSON)) + }) + + Convey("list mirror status", func() { + ms, err := boltDB.ListMirrorStatus(testWorkerIDs[0]) + So(err, ShouldBeNil) + expectedJSON, err := json.Marshal([]mirrorStatus{status1}) + So(err, ShouldBeNil) + actualJSON, err := json.Marshal(ms) + So(err, ShouldBeNil) + So(string(actualJSON), ShouldEqual, string(expectedJSON)) + }) + + Convey("list all mirror status", func() { + ms, err := boltDB.ListAllMirrorStatus() + So(err, ShouldBeNil) + expectedJSON, err := json.Marshal([]mirrorStatus{status1, status2}) + So(err, ShouldBeNil) + actualJSON, err := json.Marshal(ms) + So(err, ShouldBeNil) + So(string(actualJSON), ShouldEqual, string(expectedJSON)) + }) + + }) + + }) +} diff --git a/manager/server_test.go b/manager/server_test.go index cfc229b..8470b24 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -157,6 +157,10 @@ type mockDBAdapter struct { statusStore map[string]mirrorStatus } +func (b *mockDBAdapter) Init() error { + return nil +} + func (b *mockDBAdapter) ListWorkers() ([]worker, error) { workers := make([]worker, len(b.workerStore)) idx := 0 From 734826fa67d634dee7e986f080ebfb62f956a26f Mon Sep 17 00:00:00 2001 From: walkerning Date: Mon, 25 Apr 2016 21:49:32 +0800 Subject: [PATCH 39/66] feature(manager): worker => workerStatus --- manager/db.go | 14 +++++++------- manager/db_test.go | 2 +- manager/server.go | 4 ++-- manager/server_test.go | 20 ++++++++++---------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/manager/db.go b/manager/db.go index 36d9ff6..2403277 100644 --- a/manager/db.go +++ b/manager/db.go @@ -10,9 +10,9 @@ import ( type dbAdapter interface { Init() error - ListWorkers() ([]worker, error) - GetWorker(workerID string) (worker, error) - CreateWorker(w worker) (worker, error) + ListWorkers() ([]workerStatus, error) + GetWorker(workerID string) (workerStatus, error) + CreateWorker(w workerStatus) (workerStatus, error) UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) (mirrorStatus, error) GetMirrorStatus(workerID, mirrorID string) (mirrorStatus, error) ListMirrorStatus(workerID string) ([]mirrorStatus, error) @@ -61,11 +61,11 @@ func (b *boltAdapter) Init() (err error) { }) } -func (b *boltAdapter) ListWorkers() (ws []worker, err error) { +func (b *boltAdapter) ListWorkers() (ws []workerStatus, err error) { err = b.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(_workerBucketKey)) c := bucket.Cursor() - var w worker + var w workerStatus for k, v := c.First(); k != nil; k, v = c.Next() { jsonErr := json.Unmarshal(v, &w) if jsonErr != nil { @@ -79,7 +79,7 @@ func (b *boltAdapter) ListWorkers() (ws []worker, err error) { return } -func (b *boltAdapter) GetWorker(workerID string) (w worker, err error) { +func (b *boltAdapter) GetWorker(workerID string) (w workerStatus, err error) { err = b.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(_workerBucketKey)) v := bucket.Get([]byte(workerID)) @@ -92,7 +92,7 @@ func (b *boltAdapter) GetWorker(workerID string) (w worker, err error) { return } -func (b *boltAdapter) CreateWorker(w worker) (worker, error) { +func (b *boltAdapter) CreateWorker(w workerStatus) (workerStatus, error) { err := b.db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(_workerBucketKey)) v, err := json.Marshal(w) diff --git a/manager/db_test.go b/manager/db_test.go index e95dfe8..4c01d3e 100644 --- a/manager/db_test.go +++ b/manager/db_test.go @@ -31,7 +31,7 @@ func TestBoltAdapter(t *testing.T) { testWorkerIDs := []string{"test_worker1", "test_worker2"} Convey("create worker", func() { for _, id := range testWorkerIDs { - w := worker{ + w := workerStatus{ ID: id, Token: "token_" + id, LastOnline: time.Now(), diff --git a/manager/server.go b/manager/server.go index 38aa77f..122d5c4 100644 --- a/manager/server.go +++ b/manager/server.go @@ -21,7 +21,7 @@ const ( _infoKey = "message" ) -type worker struct { +type workerStatus struct { ID string `json:"id"` // worker name Token string `json:"token"` // session token LastOnline time.Time `json:"last_online"` // last seen @@ -72,7 +72,7 @@ func (s *managerServer) listWorkers(c *gin.Context) { // registerWorker register an newly-online worker func (s *managerServer) registerWorker(c *gin.Context) { - var _worker worker + var _worker workerStatus c.BindJSON(&_worker) newWorker, err := s.adapter.CreateWorker(_worker) if err != nil { diff --git a/manager/server_test.go b/manager/server_test.go index 8470b24..97b7e89 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -31,8 +31,8 @@ func TestHTTPServer(t *testing.T) { s := makeHTTPServer(false) So(s, ShouldNotBeNil) s.setDBAdapter(&mockDBAdapter{ - workerStore: map[string]worker{ - _magicBadWorkerID: worker{ + workerStore: map[string]workerStatus{ + _magicBadWorkerID: workerStatus{ ID: _magicBadWorkerID, }}, statusStore: make(map[string]mirrorStatus), @@ -67,7 +67,7 @@ func TestHTTPServer(t *testing.T) { }) Convey("when register a worker", func() { - w := worker{ + w := workerStatus{ ID: "test_worker1", } resp, err := postJSON(baseURL+"/workers", w) @@ -153,7 +153,7 @@ func TestHTTPServer(t *testing.T) { } type mockDBAdapter struct { - workerStore map[string]worker + workerStore map[string]workerStatus statusStore map[string]mirrorStatus } @@ -161,8 +161,8 @@ func (b *mockDBAdapter) Init() error { return nil } -func (b *mockDBAdapter) ListWorkers() ([]worker, error) { - workers := make([]worker, len(b.workerStore)) +func (b *mockDBAdapter) ListWorkers() ([]workerStatus, error) { + workers := make([]workerStatus, len(b.workerStore)) idx := 0 for _, w := range b.workerStore { workers[idx] = w @@ -171,18 +171,18 @@ func (b *mockDBAdapter) ListWorkers() ([]worker, error) { return workers, nil } -func (b *mockDBAdapter) GetWorker(workerID string) (worker, error) { +func (b *mockDBAdapter) GetWorker(workerID string) (workerStatus, error) { w, ok := b.workerStore[workerID] if !ok { - return worker{}, fmt.Errorf("invalid workerId") + return workerStatus{}, fmt.Errorf("invalid workerId") } return w, nil } -func (b *mockDBAdapter) CreateWorker(w worker) (worker, error) { +func (b *mockDBAdapter) CreateWorker(w workerStatus) (workerStatus, error) { // _, ok := b.workerStore[w.ID] // if ok { - // return worker{}, fmt.Errorf("duplicate worker name") + // return workerStatus{}, fmt.Errorf("duplicate worker name") // } b.workerStore[w.ID] = w return w, nil From daa0b3c204da0b3ef46df87db578726e14c5b022 Mon Sep 17 00:00:00 2001 From: walkerning Date: Tue, 26 Apr 2016 12:01:34 +0800 Subject: [PATCH 40/66] refactor(manager): command pulling to command pushing and tests --- manager/middleware.go | 4 +- manager/server.go | 71 +++++------------------------- manager/server_test.go | 99 +++++++++++++++++++++++++++++++++++------- manager/status.go | 7 +++ manager/util.go | 13 ++++++ 5 files changed, 119 insertions(+), 75 deletions(-) create mode 100644 manager/util.go diff --git a/manager/middleware.go b/manager/middleware.go index f620261..3c2d1ea 100644 --- a/manager/middleware.go +++ b/manager/middleware.go @@ -11,7 +11,9 @@ func contextErrorLogger(c *gin.Context) { errs := c.Errors.ByType(gin.ErrorTypeAny) if len(errs) > 0 { for _, err := range errs { - logger.Error(`"in request "%s %s: %s"`, c.Request.Method, c.Request.URL.Path, err.Error()) + logger.Error(`"in request "%s %s: %s"`, + c.Request.Method, c.Request.URL.Path, + err.Error()) } } // pass on to the next middleware in chain diff --git a/manager/server.go b/manager/server.go index 122d5c4..b8fe5a8 100644 --- a/manager/server.go +++ b/manager/server.go @@ -3,35 +3,17 @@ package manager import ( "fmt" "net/http" - "sync" - "time" "github.com/gin-gonic/gin" . "github.com/tuna/tunasync/internal" ) -const ( - maxQueuedCmdNum = 3 - cmdPollTime = 10 * time.Second -) - const ( _errorKey = "error" _infoKey = "message" ) -type workerStatus struct { - ID string `json:"id"` // worker name - Token string `json:"token"` // session token - LastOnline time.Time `json:"last_online"` // last seen -} - -var ( - workerChannelMu sync.RWMutex - workerChannels = make(map[string]chan WorkerCmd) -) - type managerServer struct { *gin.Engine adapter dbAdapter @@ -84,9 +66,6 @@ func (s *managerServer) registerWorker(c *gin.Context) { return } // create workerCmd channel for this worker - workerChannelMu.Lock() - defer workerChannelMu.Unlock() - workerChannels[_worker.ID] = make(chan WorkerCmd, maxQueuedCmdNum) c.JSON(http.StatusOK, newWorker) } @@ -129,11 +108,8 @@ func (s *managerServer) updateJobOfWorker(c *gin.Context) { } func (s *managerServer) handleClientCmd(c *gin.Context) { - workerChannelMu.RLock() - defer workerChannelMu.RUnlock() var clientCmd ClientCmd c.BindJSON(&clientCmd) - // TODO: decide which worker should do this mirror when WorkerID is null string workerID := clientCmd.WorkerID if workerID == "" { // TODO: decide which worker should do this mirror when WorkerID is null string @@ -142,50 +118,30 @@ func (s *managerServer) handleClientCmd(c *gin.Context) { return } - workerChannel, ok := workerChannels[workerID] - if !ok { + w, err := s.adapter.GetWorker(workerID) + if err != nil { err := fmt.Errorf("worker %s is not registered yet", workerID) s.returnErrJSON(c, http.StatusBadRequest, err) return } + workerURL := w.URL // parse client cmd into worker cmd workerCmd := WorkerCmd{ Cmd: clientCmd.Cmd, MirrorID: clientCmd.MirrorID, Args: clientCmd.Args, } - select { - case workerChannel <- workerCmd: - // successfully insert command to channel - c.JSON(http.StatusOK, struct{}{}) - default: - // pending commands for that worker exceed - // the maxQueuedCmdNum threshold - err := fmt.Errorf("pending commands for worker %s exceed"+ - "the %d threshold, the command is dropped", - workerID, maxQueuedCmdNum) + + // post command to worker + _, err = postJSON(workerURL, workerCmd) + if err != nil { + err := fmt.Errorf("post command to worker %s(%s) fail: %s", workerID, workerURL, err.Error()) c.Error(err) - s.returnErrJSON(c, http.StatusServiceUnavailable, err) + s.returnErrJSON(c, http.StatusInternalServerError, err) return } -} - -func (s *managerServer) getCmdOfWorker(c *gin.Context) { - workerID := c.Param("id") - workerChannelMu.RLock() - defer workerChannelMu.RUnlock() - - workerChannel := workerChannels[workerID] - for { - select { - case _ = <-workerChannel: - // TODO: push new command to worker client - continue - case <-time.After(cmdPollTime): - // time limit exceeded, close the connection - break - } - } + // TODO: check response for success + c.JSON(http.StatusOK, gin.H{_infoKey: "successfully send command to worker " + workerID}) } func (s *managerServer) setDBAdapter(adapter dbAdapter) { @@ -223,11 +179,8 @@ func makeHTTPServer(debug bool) *managerServer { // post job status workerValidateGroup.POST(":id/jobs/:job", s.updateJobOfWorker) - // worker command polling - workerValidateGroup.GET(":id/cmd_stream", s.getCmdOfWorker) - // for tunasynctl to post commands - s.POST("/cmd/", s.handleClientCmd) + s.POST("/cmd", s.handleClientCmd) return s } diff --git a/manager/server_test.go b/manager/server_test.go index 97b7e89..6a14d9b 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -1,7 +1,6 @@ package manager import ( - "bytes" "encoding/json" "fmt" "io/ioutil" @@ -11,6 +10,8 @@ import ( "testing" "time" + "github.com/gin-gonic/gin" + . "github.com/smartystreets/goconvey/convey" . "github.com/tuna/tunasync/internal" ) @@ -19,14 +20,8 @@ const ( _magicBadWorkerID = "magic_bad_worker_id" ) -func postJSON(url string, obj interface{}) (*http.Response, error) { - b := new(bytes.Buffer) - json.NewEncoder(b).Encode(obj) - return http.Post(url, "application/json; charset=utf-8", b) -} - func TestHTTPServer(t *testing.T) { - Convey("HTTP server should work", t, func() { + Convey("HTTP server should work", t, func(ctx C) { InitLogger(true, true, false) s := makeHTTPServer(false) So(s, ShouldNotBeNil) @@ -55,7 +50,7 @@ func TestHTTPServer(t *testing.T) { So(err, ShouldBeNil) So(p[_infoKey], ShouldEqual, "pong") - Convey("when database fail", func() { + Convey("when database fail", func(ctx C) { resp, err := http.Get(fmt.Sprintf("%s/workers/%s/jobs", baseURL, _magicBadWorkerID)) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError) @@ -66,7 +61,7 @@ func TestHTTPServer(t *testing.T) { So(msg[_errorKey], ShouldEqual, fmt.Sprintf("failed to list jobs of worker %s: %s", _magicBadWorkerID, "database fail")) }) - Convey("when register a worker", func() { + Convey("when register a worker", func(ctx C) { w := workerStatus{ ID: "test_worker1", } @@ -74,7 +69,7 @@ func TestHTTPServer(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) - Convey("list all workers", func() { + Convey("list all workers", func(ctx C) { So(err, ShouldBeNil) resp, err := http.Get(baseURL + "/workers") So(err, ShouldBeNil) @@ -85,7 +80,7 @@ func TestHTTPServer(t *testing.T) { So(len(actualResponseObj), ShouldEqual, 2) }) - Convey("update mirror status of a existed worker", func() { + Convey("update mirror status of a existed worker", func(ctx C) { status := mirrorStatus{ Name: "arch-sync1", Worker: "test_worker1", @@ -96,10 +91,11 @@ func TestHTTPServer(t *testing.T) { Size: "3GB", } resp, err := postJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", baseURL, status.Worker, status.Name), status) + defer resp.Body.Close() So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) - Convey("list mirror status of an existed worker", func() { + Convey("list mirror status of an existed worker", func(ctx C) { expectedResponse, err := json.Marshal([]mirrorStatus{status}) So(err, ShouldBeNil) @@ -113,7 +109,7 @@ func TestHTTPServer(t *testing.T) { So(strings.TrimSpace(string(body)), ShouldEqual, string(expectedResponse)) }) - Convey("list all job status of all workers", func() { + Convey("list all job status of all workers", func(ctx C) { expectedResponse, err := json.Marshal([]mirrorStatus{status}) So(err, ShouldBeNil) resp, err := http.Get(baseURL + "/jobs") @@ -127,7 +123,7 @@ func TestHTTPServer(t *testing.T) { }) }) - Convey("update mirror status of an inexisted worker", func() { + Convey("update mirror status of an inexisted worker", func(ctx C) { invalidWorker := "test_worker2" status := mirrorStatus{ Name: "arch-sync2", @@ -148,6 +144,65 @@ func TestHTTPServer(t *testing.T) { So(err, ShouldBeNil) So(msg[_errorKey], ShouldEqual, "invalid workerID "+invalidWorker) }) + Convey("handle client command", func(ctx C) { + cmdChan := make(chan WorkerCmd, 1) + workerServer := makeMockWorkerServer(cmdChan) + workerPort := rand.Intn(10000) + 30000 + bindAddress := fmt.Sprintf("127.0.0.1:%d", workerPort) + workerBaseURL := fmt.Sprintf("http://%s", bindAddress) + w := workerStatus{ + ID: "test_worker_cmd", + URL: workerBaseURL + "/cmd", + } + resp, err := postJSON(baseURL+"/workers", w) + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + + go func() { + // run the mock worker server + workerServer.Run(bindAddress) + }() + time.Sleep(50 * time.Microsecond) + // verify the worker mock server is running + workerResp, err := http.Get(workerBaseURL + "/ping") + defer workerResp.Body.Close() + So(err, ShouldBeNil) + So(workerResp.StatusCode, ShouldEqual, http.StatusOK) + + Convey("when client send wrong cmd", func(ctx C) { + clientCmd := ClientCmd{ + Cmd: CmdStart, + MirrorID: "ubuntu-sync", + WorkerID: "not_exist_worker", + } + resp, err := postJSON(baseURL+"/cmd", clientCmd) + defer resp.Body.Close() + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusBadRequest) + }) + + Convey("when client send correct cmd", func(ctx C) { + clientCmd := ClientCmd{ + Cmd: CmdStart, + MirrorID: "ubuntu-sync", + WorkerID: w.ID, + } + + resp, err := postJSON(baseURL+"/cmd", clientCmd) + defer resp.Body.Close() + + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + time.Sleep(50 * time.Microsecond) + select { + case cmd := <-cmdChan: + ctx.So(cmd.Cmd, ShouldEqual, clientCmd.Cmd) + ctx.So(cmd.MirrorID, ShouldEqual, clientCmd.MirrorID) + default: + ctx.So(0, ShouldEqual, 1) + } + }) + }) }) }) } @@ -233,3 +288,17 @@ func (b *mockDBAdapter) ListAllMirrorStatus() ([]mirrorStatus, error) { func (b *mockDBAdapter) Close() error { return nil } + +func makeMockWorkerServer(cmdChan chan WorkerCmd) *gin.Engine { + r := gin.Default() + r.GET("/ping", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{_infoKey: "pong"}) + }) + r.POST("/cmd", func(c *gin.Context) { + var cmd WorkerCmd + c.BindJSON(&cmd) + cmdChan <- cmd + }) + + return r +} diff --git a/manager/status.go b/manager/status.go index 4708163..8c6150c 100644 --- a/manager/status.go +++ b/manager/status.go @@ -106,3 +106,10 @@ func (s *mirrorStatus) UnmarshalJSON(v []byte) error { } return nil } + +type workerStatus struct { + ID string `json:"id"` // worker name + Token string `json:"token"` // session token + URL string `json:"url"` // worker url + LastOnline time.Time `json:"last_online"` // last seen +} diff --git a/manager/util.go b/manager/util.go new file mode 100644 index 0000000..4174eea --- /dev/null +++ b/manager/util.go @@ -0,0 +1,13 @@ +package manager + +import ( + "bytes" + "encoding/json" + "net/http" +) + +func postJSON(url string, obj interface{}) (*http.Response, error) { + b := new(bytes.Buffer) + json.NewEncoder(b).Encode(obj) + return http.Post(url, "application/json; charset=utf-8", b) +} From ce3471e30d1739dbb131ba58aed07d05091afd7b Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 18:34:22 +0800 Subject: [PATCH 41/66] feature(worker): implemented Worker object, worker side code is almost done --- internal/msg.go | 6 +- internal/util.go | 79 +++++++ manager/server.go | 5 +- worker/cmd_provider.go | 4 + worker/config_test.go | 13 +- worker/job.go | 12 +- worker/job_test.go | 6 +- worker/loglimit_test.go | 2 +- worker/main.go | 108 --------- worker/provider.go | 1 + worker/rsync_provider.go | 4 + worker/two_stage_rsync_provider.go | 4 + worker/worker.go | 342 +++++++++++++++++++++++++++++ 13 files changed, 462 insertions(+), 124 deletions(-) create mode 100644 internal/util.go delete mode 100644 worker/main.go create mode 100644 worker/worker.go diff --git a/internal/msg.go b/internal/msg.go index b1478f2..a4e2838 100644 --- a/internal/msg.go +++ b/internal/msg.go @@ -4,7 +4,7 @@ import "time" // A StatusUpdateMsg represents a msg when // a worker has done syncing -type StatusUpdateMsg struct { +type MirrorStatus struct { Name string `json:"name"` Worker string `json:"worker"` IsMaster bool `json:"is_master"` @@ -19,7 +19,9 @@ type StatusUpdateMsg struct { // a worker, and sent from the manager to clients. type WorkerInfoMsg struct { ID string `json:"id"` - LastOnline time.Time `json:"last_online"` + URL string `json:"url"` // worker url + Token string `json:"token"` // session token + LastOnline time.Time `json:"last_online"` // last seen } type CmdVerb uint8 diff --git a/internal/util.go b/internal/util.go new file mode 100644 index 0000000..9773914 --- /dev/null +++ b/internal/util.go @@ -0,0 +1,79 @@ +package internal + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "io/ioutil" + "net/http" +) + +// GetTLSConfig generate tls.Config from CAFile +func GetTLSConfig(CAFile string) (*tls.Config, error) { + caCert, err := ioutil.ReadFile(CAFile) + if err != nil { + return nil, err + } + caCertPool := x509.NewCertPool() + if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { + return nil, errors.New("Failed to add CA to pool") + } + + tlsConfig := &tls.Config{ + RootCAs: caCertPool, + } + tlsConfig.BuildNameToCertificate() + return tlsConfig, nil +} + +// PostJSON posts json object to url +func PostJSON(url string, obj interface{}, tlsConfig *tls.Config) (*http.Response, error) { + var client *http.Client + if tlsConfig == nil { + client = &http.Client{} + } else { + tr := &http.Transport{ + TLSClientConfig: tlsConfig, + } + client = &http.Client{ + Transport: tr, + } + } + + b := new(bytes.Buffer) + if err := json.NewEncoder(b).Encode(obj); err != nil { + return nil, err + } + return client.Post(url, "application/json; charset=utf-8", b) +} + +// GetJSON gets a json response from url +func GetJSON(url string, obj interface{}, tlsConfig *tls.Config) (*http.Response, error) { + var client *http.Client + if tlsConfig == nil { + client = &http.Client{} + } else { + tr := &http.Transport{ + TLSClientConfig: tlsConfig, + } + client = &http.Client{ + Transport: tr, + } + } + + resp, err := client.Get(url) + if err != nil { + return resp, err + } + if resp.StatusCode != http.StatusOK { + return resp, errors.New("HTTP status code is not 200") + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return resp, err + } + return resp, json.Unmarshal(body, obj) +} diff --git a/manager/server.go b/manager/server.go index b8fe5a8..5102aae 100644 --- a/manager/server.go +++ b/manager/server.go @@ -47,7 +47,10 @@ func (s *managerServer) listWorkers(c *gin.Context) { } for _, w := range workers { workerInfos = append(workerInfos, - WorkerInfoMsg{w.ID, w.LastOnline}) + WorkerInfoMsg{ + ID: w.ID, + LastOnline: w.LastOnline, + }) } c.JSON(http.StatusOK, workerInfos) } diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go index 38cdcac..8ed8f9e 100644 --- a/worker/cmd_provider.go +++ b/worker/cmd_provider.go @@ -44,6 +44,10 @@ func newCmdProvider(c cmdConfig) (*cmdProvider, error) { return provider, nil } +func (p *cmdProvider) Upstream() string { + return p.upstreamURL +} + func (p *cmdProvider) Run() error { if err := p.Start(); err != nil { return err diff --git a/worker/config_test.go b/worker/config_test.go index ea461a2..94d8bae 100644 --- a/worker/config_test.go +++ b/worker/config_test.go @@ -110,16 +110,21 @@ exclude_file = "/etc/tunasync.d/fedora-exclude.txt" cfg, err := loadConfig(tmpfile.Name()) So(err, ShouldBeNil) - providers := initProviders(cfg) + w := &Worker{ + cfg: cfg, + providers: make(map[string]mirrorProvider), + } - p := providers[0] + w.initProviders() + + p := w.providers["AOSP"] So(p.Name(), ShouldEqual, "AOSP") So(p.LogDir(), ShouldEqual, "/var/log/tunasync/AOSP") So(p.LogFile(), ShouldEqual, "/var/log/tunasync/AOSP/latest.log") _, ok := p.(*cmdProvider) So(ok, ShouldBeTrue) - p = providers[1] + p = w.providers["debian"] So(p.Name(), ShouldEqual, "debian") So(p.LogDir(), ShouldEqual, "/var/log/tunasync/debian") So(p.LogFile(), ShouldEqual, "/var/log/tunasync/debian/latest.log") @@ -128,7 +133,7 @@ exclude_file = "/etc/tunasync.d/fedora-exclude.txt" So(r2p.stage1Profile, ShouldEqual, "debian") So(r2p.WorkingDir(), ShouldEqual, "/data/mirrors/debian") - p = providers[2] + p = w.providers["fedora"] So(p.Name(), ShouldEqual, "fedora") So(p.LogDir(), ShouldEqual, "/var/log/tunasync/fedora") So(p.LogFile(), ShouldEqual, "/var/log/tunasync/fedora/latest.log") diff --git a/worker/job.go b/worker/job.go index d94cc3c..f71136a 100644 --- a/worker/job.go +++ b/worker/job.go @@ -28,7 +28,7 @@ type jobMessage struct { type mirrorJob struct { provider mirrorProvider ctrlChan chan ctrlAction - stopped chan empty + disabled chan empty enabled bool } @@ -44,12 +44,12 @@ func (m *mirrorJob) Name() string { return m.provider.Name() } -func (m *mirrorJob) Stopped() bool { +func (m *mirrorJob) Disabled() bool { if !m.enabled { return true } select { - case <-m.stopped: + case <-m.disabled: return true default: return false @@ -65,8 +65,8 @@ func (m *mirrorJob) Stopped() bool { // TODO: message struct for managerChan func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) error { - m.stopped = make(chan empty) - defer close(m.stopped) + m.disabled = make(chan empty) + defer close(m.disabled) provider := m.provider @@ -210,6 +210,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err close(kill) <-jobDone case jobDisable: + m.enabled = false close(kill) <-jobDone return nil @@ -234,6 +235,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err case jobStop: m.enabled = false case jobDisable: + m.enabled = false return nil case jobRestart: m.enabled = true diff --git a/worker/job_test.go b/worker/job_test.go index 15bc716..1edc14b 100644 --- a/worker/job_test.go +++ b/worker/job_test.go @@ -105,7 +105,7 @@ func TestMirrorJob(t *testing.T) { select { case <-managerChan: So(0, ShouldEqual, 1) // made this fail - case <-job.stopped: + case <-job.disabled: So(0, ShouldEqual, 0) } }) @@ -145,7 +145,7 @@ echo $TUNASYNC_WORKING_DIR So(err, ShouldBeNil) So(string(loggedContent), ShouldEqual, exceptedOutput) job.ctrlChan <- jobDisable - <-job.stopped + <-job.disabled }) Convey("If we don't kill it", func(ctx C) { @@ -168,7 +168,7 @@ echo $TUNASYNC_WORKING_DIR So(err, ShouldBeNil) So(string(loggedContent), ShouldEqual, exceptedOutput) job.ctrlChan <- jobDisable - <-job.stopped + <-job.disabled }) }) diff --git a/worker/loglimit_test.go b/worker/loglimit_test.go index 74d87b6..e42a78a 100644 --- a/worker/loglimit_test.go +++ b/worker/loglimit_test.go @@ -122,7 +122,7 @@ sleep 5 So(msg.status, ShouldEqual, Failed) job.ctrlChan <- jobDisable - <-job.stopped + <-job.disabled So(logFile, ShouldNotEqual, provider.LogFile()) diff --git a/worker/main.go b/worker/main.go deleted file mode 100644 index a51f579..0000000 --- a/worker/main.go +++ /dev/null @@ -1,108 +0,0 @@ -package worker - -import ( - "bytes" - "errors" - "html/template" - "path/filepath" - "time" -) - -// toplevel module for workers - -func initProviders(c *Config) []mirrorProvider { - - formatLogDir := func(logDir string, m mirrorConfig) string { - tmpl, err := template.New("logDirTmpl-" + m.Name).Parse(logDir) - if err != nil { - panic(err) - } - var formatedLogDir bytes.Buffer - tmpl.Execute(&formatedLogDir, m) - return formatedLogDir.String() - } - - providers := []mirrorProvider{} - - for _, mirror := range c.Mirrors { - logDir := mirror.LogDir - mirrorDir := mirror.MirrorDir - if logDir == "" { - logDir = c.Global.LogDir - } - if mirrorDir == "" { - mirrorDir = c.Global.MirrorDir - } - logDir = formatLogDir(logDir, mirror) - switch mirror.Provider { - case ProvCommand: - pc := cmdConfig{ - name: mirror.Name, - upstreamURL: mirror.Upstream, - command: mirror.Command, - workingDir: filepath.Join(mirrorDir, mirror.Name), - logDir: logDir, - logFile: filepath.Join(logDir, "latest.log"), - interval: time.Duration(mirror.Interval) * time.Minute, - env: mirror.Env, - } - p, err := newCmdProvider(pc) - if err != nil { - panic(err) - } - providers = append(providers, p) - case ProvRsync: - rc := rsyncConfig{ - name: mirror.Name, - upstreamURL: mirror.Upstream, - password: mirror.Password, - excludeFile: mirror.ExcludeFile, - workingDir: filepath.Join(mirrorDir, mirror.Name), - logDir: logDir, - logFile: filepath.Join(logDir, "latest.log"), - useIPv6: mirror.UseIPv6, - interval: time.Duration(mirror.Interval) * time.Minute, - } - p, err := newRsyncProvider(rc) - if err != nil { - panic(err) - } - providers = append(providers, p) - case ProvTwoStageRsync: - rc := twoStageRsyncConfig{ - name: mirror.Name, - stage1Profile: mirror.Stage1Profile, - upstreamURL: mirror.Upstream, - password: mirror.Password, - excludeFile: mirror.ExcludeFile, - workingDir: filepath.Join(mirrorDir, mirror.Name), - logDir: logDir, - logFile: filepath.Join(logDir, "latest.log"), - useIPv6: mirror.UseIPv6, - interval: time.Duration(mirror.Interval) * time.Minute, - } - p, err := newTwoStageRsyncProvider(rc) - if err != nil { - panic(err) - } - providers = append(providers, p) - default: - panic(errors.New("Invalid mirror provider")) - - } - - } - return providers -} - -func main() { - - for { - // if time.Now().After() { - // - // } - - time.Sleep(1 * time.Second) - } - -} diff --git a/worker/provider.go b/worker/provider.go index 6aba4dc..498a3e7 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -21,6 +21,7 @@ const ( type mirrorProvider interface { // name Name() string + Upstream() string // run mirror job in background Run() error diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go index 96e88c7..49153c9 100644 --- a/worker/rsync_provider.go +++ b/worker/rsync_provider.go @@ -63,6 +63,10 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) { return provider, nil } +func (p *rsyncProvider) Upstream() string { + return p.upstreamURL +} + func (p *rsyncProvider) Run() error { if err := p.Start(); err != nil { return err diff --git a/worker/two_stage_rsync_provider.go b/worker/two_stage_rsync_provider.go index 247aa0e..5a53716 100644 --- a/worker/two_stage_rsync_provider.go +++ b/worker/two_stage_rsync_provider.go @@ -70,6 +70,10 @@ func newTwoStageRsyncProvider(c twoStageRsyncConfig) (*twoStageRsyncProvider, er return provider, nil } +func (p *twoStageRsyncProvider) Upstream() string { + return p.upstreamURL +} + func (p *twoStageRsyncProvider) Options(stage int) ([]string, error) { var options []string if stage == 1 { diff --git a/worker/worker.go b/worker/worker.go new file mode 100644 index 0000000..063f932 --- /dev/null +++ b/worker/worker.go @@ -0,0 +1,342 @@ +package worker + +import ( + "bytes" + "crypto/tls" + "errors" + "fmt" + "html/template" + "net/http" + "path/filepath" + "time" + + "github.com/gin-gonic/gin" + . "github.com/tuna/tunasync/internal" +) + +var tunasyncWorker *Worker + +// A Worker is a instance of tunasync worker +type Worker struct { + cfg *Config + providers map[string]mirrorProvider + jobs map[string]*mirrorJob + + managerChan chan jobMessage + semaphore chan empty + + schedule *scheduleQueue + httpServer *gin.Engine + tlsConfig *tls.Config + + mirrorStatus map[string]SyncStatus +} + +// GetTUNASyncWorker returns a singalton worker +func GetTUNASyncWorker(cfg *Config) *Worker { + if tunasyncWorker != nil { + return tunasyncWorker + } + + w := &Worker{ + cfg: cfg, + providers: make(map[string]mirrorProvider), + jobs: make(map[string]*mirrorJob), + + managerChan: make(chan jobMessage, 32), + semaphore: make(chan empty, cfg.Global.Concurrent), + + schedule: newScheduleQueue(), + mirrorStatus: make(map[string]SyncStatus), + } + w.initJobs() + w.makeHTTPServer() + tunasyncWorker = w + return w +} + +func (w *Worker) initProviders() { + c := w.cfg + + formatLogDir := func(logDir string, m mirrorConfig) string { + tmpl, err := template.New("logDirTmpl-" + m.Name).Parse(logDir) + if err != nil { + panic(err) + } + var formatedLogDir bytes.Buffer + tmpl.Execute(&formatedLogDir, m) + return formatedLogDir.String() + } + + for _, mirror := range c.Mirrors { + logDir := mirror.LogDir + mirrorDir := mirror.MirrorDir + if logDir == "" { + logDir = c.Global.LogDir + } + if mirrorDir == "" { + mirrorDir = c.Global.MirrorDir + } + logDir = formatLogDir(logDir, mirror) + + var provider mirrorProvider + + switch mirror.Provider { + case ProvCommand: + pc := cmdConfig{ + name: mirror.Name, + upstreamURL: mirror.Upstream, + command: mirror.Command, + workingDir: filepath.Join(mirrorDir, mirror.Name), + logDir: logDir, + logFile: filepath.Join(logDir, "latest.log"), + interval: time.Duration(mirror.Interval) * time.Minute, + env: mirror.Env, + } + p, err := newCmdProvider(pc) + if err != nil { + panic(err) + } + provider = p + case ProvRsync: + rc := rsyncConfig{ + name: mirror.Name, + upstreamURL: mirror.Upstream, + password: mirror.Password, + excludeFile: mirror.ExcludeFile, + workingDir: filepath.Join(mirrorDir, mirror.Name), + logDir: logDir, + logFile: filepath.Join(logDir, "latest.log"), + useIPv6: mirror.UseIPv6, + interval: time.Duration(mirror.Interval) * time.Minute, + } + p, err := newRsyncProvider(rc) + if err != nil { + panic(err) + } + provider = p + case ProvTwoStageRsync: + rc := twoStageRsyncConfig{ + name: mirror.Name, + stage1Profile: mirror.Stage1Profile, + upstreamURL: mirror.Upstream, + password: mirror.Password, + excludeFile: mirror.ExcludeFile, + workingDir: filepath.Join(mirrorDir, mirror.Name), + logDir: logDir, + logFile: filepath.Join(logDir, "latest.log"), + useIPv6: mirror.UseIPv6, + interval: time.Duration(mirror.Interval) * time.Minute, + } + p, err := newTwoStageRsyncProvider(rc) + if err != nil { + panic(err) + } + provider = p + default: + panic(errors.New("Invalid mirror provider")) + + } + + provider.AddHook(newLogLimiter(provider)) + w.providers[provider.Name()] = provider + + } +} + +func (w *Worker) initJobs() { + w.initProviders() + + for name, provider := range w.providers { + w.jobs[name] = newMirrorJob(provider) + go w.jobs[name].Run(w.managerChan, w.semaphore) + w.mirrorStatus[name] = Paused + } +} + +// Ctrl server receives commands from the manager +func (w *Worker) makeHTTPServer() { + s := gin.New() + s.Use(gin.Recovery()) + + s.POST("/", func(c *gin.Context) { + var cmd WorkerCmd + + if err := c.BindJSON(&cmd); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"msg": "Invalid request"}) + return + } + job, ok := w.jobs[cmd.MirrorID] + if !ok { + c.JSON(http.StatusNotFound, gin.H{"msg": fmt.Sprintf("Mirror ``%s'' not found", cmd.MirrorID)}) + return + } + // if job disabled, start them first + switch cmd.Cmd { + case CmdStart, CmdRestart: + if job.Disabled() { + go job.Run(w.managerChan, w.semaphore) + } + } + switch cmd.Cmd { + case CmdStart: + job.ctrlChan <- jobStart + case CmdStop: + job.ctrlChan <- jobStop + case CmdRestart: + job.ctrlChan <- jobRestart + case CmdDisable: + w.schedule.Remove(job.Name()) + job.ctrlChan <- jobDisable + <-job.disabled + case CmdPing: + job.ctrlChan <- jobStart + default: + c.JSON(http.StatusNotAcceptable, gin.H{"msg": "Invalid Command"}) + return + } + + c.JSON(http.StatusOK, gin.H{"msg": "OK"}) + }) + + w.httpServer = s +} + +func (w *Worker) runHTTPServer() { + addr := fmt.Sprintf("%s:%d", w.cfg.Server.Addr, w.cfg.Server.Port) + + if w.cfg.Server.SSLCert == "" && w.cfg.Server.SSLKey == "" { + if err := w.httpServer.Run(addr); err != nil { + panic(err) + } + } else { + if err := w.httpServer.RunTLS(addr, w.cfg.Server.SSLCert, w.cfg.Server.SSLKey); err != nil { + panic(err) + } + } +} + +// Run runs worker forever +func (w *Worker) Run() { + w.registorWorker() + go w.runHTTPServer() + w.runSchedule() +} + +func (w *Worker) runSchedule() { + mirrorList := w.fetchJobStatus() + unset := make(map[string]bool) + for name := range w.jobs { + unset[name] = true + } + for _, m := range mirrorList { + if job, ok := w.jobs[m.Name]; ok { + stime := m.LastUpdate.Add(job.provider.Interval()) + w.schedule.AddJob(stime, job) + delete(unset, m.Name) + } + } + for name := range unset { + job := w.jobs[name] + w.schedule.AddJob(time.Now(), job) + } + + for { + select { + case jobMsg := <-w.managerChan: + // got status update from job + w.updateStatus(jobMsg) + status := w.mirrorStatus[jobMsg.name] + if status == Disabled || status == Paused { + continue + } + w.mirrorStatus[jobMsg.name] = jobMsg.status + switch jobMsg.status { + case Success, Failed: + job := w.jobs[jobMsg.name] + w.schedule.AddJob( + time.Now().Add(job.provider.Interval()), + job, + ) + } + + case <-time.Tick(10 * time.Second): + if job := w.schedule.Pop(); job != nil { + job.ctrlChan <- jobStart + } + } + + } + +} + +// Name returns worker name +func (w *Worker) Name() string { + return w.cfg.Global.Name +} + +// URL returns the url to http server of the worker +func (w *Worker) URL() string { + proto := "https" + if w.cfg.Server.SSLCert == "" && w.cfg.Server.SSLKey == "" { + proto = "http" + } + + return fmt.Sprintf("%s://%s:%d/", proto, w.cfg.Server.Hostname, w.cfg.Server.Port) +} + +func (w *Worker) registorWorker() { + url := fmt.Sprintf( + "%s/workers", + w.cfg.Manager.APIBase, + ) + + msg := WorkerInfoMsg{ + ID: w.Name(), + URL: w.URL(), + } + + if _, err := PostJSON(url, msg, w.tlsConfig); err != nil { + logger.Error("Failed to register worker") + } +} + +func (w *Worker) updateStatus(jobMsg jobMessage) { + url := fmt.Sprintf( + "%s/%s/jobs/%s", + w.cfg.Manager.APIBase, + w.Name(), + jobMsg.name, + ) + p := w.providers[jobMsg.name] + smsg := MirrorStatus{ + Name: jobMsg.name, + Worker: w.cfg.Global.Name, + IsMaster: true, + Status: jobMsg.status, + LastUpdate: time.Now(), + Upstream: p.Upstream(), + Size: "unknown", + ErrorMsg: jobMsg.msg, + } + + if _, err := PostJSON(url, smsg, w.tlsConfig); err != nil { + logger.Error("Failed to update mirror(%s) status: %s", jobMsg.name, err.Error()) + } +} + +func (w *Worker) fetchJobStatus() []MirrorStatus { + var mirrorList []MirrorStatus + + url := fmt.Sprintf( + "%s/%s/jobs", + w.cfg.Manager.APIBase, + w.Name(), + ) + + if _, err := GetJSON(url, &mirrorList, w.tlsConfig); err != nil { + logger.Error("Failed to fetch job status: %s", err.Error()) + } + + return mirrorList +} From 0dcd89da31b220fd0cdaf87c1eeff53431dd21ee Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 19:21:41 +0800 Subject: [PATCH 42/66] refactor(manager): refactored structure names in manager --- internal/msg.go | 4 +- manager/db.go | 36 +++++------ manager/db_test.go | 10 +-- manager/server.go | 17 ++++-- manager/server_test.go | 54 +++++++++-------- manager/status.go | 135 +++++++++++++---------------------------- manager/status_test.go | 19 +++--- worker/worker.go | 2 +- 8 files changed, 119 insertions(+), 158 deletions(-) diff --git a/internal/msg.go b/internal/msg.go index a4e2838..a433949 100644 --- a/internal/msg.go +++ b/internal/msg.go @@ -15,9 +15,9 @@ type MirrorStatus struct { ErrorMsg string `json:"error_msg"` } -// A WorkerInfoMsg is the information struct that describe +// A WorkerStatus is the information struct that describe // a worker, and sent from the manager to clients. -type WorkerInfoMsg struct { +type WorkerStatus struct { ID string `json:"id"` URL string `json:"url"` // worker url Token string `json:"token"` // session token diff --git a/manager/db.go b/manager/db.go index 2403277..42623a0 100644 --- a/manager/db.go +++ b/manager/db.go @@ -6,17 +6,19 @@ import ( "strings" "github.com/boltdb/bolt" + + . "github.com/tuna/tunasync/internal" ) type dbAdapter interface { Init() error - ListWorkers() ([]workerStatus, error) - GetWorker(workerID string) (workerStatus, error) - CreateWorker(w workerStatus) (workerStatus, error) - UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) (mirrorStatus, error) - GetMirrorStatus(workerID, mirrorID string) (mirrorStatus, error) - ListMirrorStatus(workerID string) ([]mirrorStatus, error) - ListAllMirrorStatus() ([]mirrorStatus, error) + ListWorkers() ([]WorkerStatus, error) + GetWorker(workerID string) (WorkerStatus, error) + CreateWorker(w WorkerStatus) (WorkerStatus, error) + UpdateMirrorStatus(workerID, mirrorID string, status MirrorStatus) (MirrorStatus, error) + GetMirrorStatus(workerID, mirrorID string) (MirrorStatus, error) + ListMirrorStatus(workerID string) ([]MirrorStatus, error) + ListAllMirrorStatus() ([]MirrorStatus, error) Close() error } @@ -61,11 +63,11 @@ func (b *boltAdapter) Init() (err error) { }) } -func (b *boltAdapter) ListWorkers() (ws []workerStatus, err error) { +func (b *boltAdapter) ListWorkers() (ws []WorkerStatus, err error) { err = b.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(_workerBucketKey)) c := bucket.Cursor() - var w workerStatus + var w WorkerStatus for k, v := c.First(); k != nil; k, v = c.Next() { jsonErr := json.Unmarshal(v, &w) if jsonErr != nil { @@ -79,7 +81,7 @@ func (b *boltAdapter) ListWorkers() (ws []workerStatus, err error) { return } -func (b *boltAdapter) GetWorker(workerID string) (w workerStatus, err error) { +func (b *boltAdapter) GetWorker(workerID string) (w WorkerStatus, err error) { err = b.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(_workerBucketKey)) v := bucket.Get([]byte(workerID)) @@ -92,7 +94,7 @@ func (b *boltAdapter) GetWorker(workerID string) (w workerStatus, err error) { return } -func (b *boltAdapter) CreateWorker(w workerStatus) (workerStatus, error) { +func (b *boltAdapter) CreateWorker(w WorkerStatus) (WorkerStatus, error) { err := b.db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(_workerBucketKey)) v, err := json.Marshal(w) @@ -105,7 +107,7 @@ func (b *boltAdapter) CreateWorker(w workerStatus) (workerStatus, error) { return w, err } -func (b *boltAdapter) UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) (mirrorStatus, error) { +func (b *boltAdapter) UpdateMirrorStatus(workerID, mirrorID string, status MirrorStatus) (MirrorStatus, error) { id := mirrorID + "/" + workerID err := b.db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(_statusBucketKey)) @@ -116,7 +118,7 @@ func (b *boltAdapter) UpdateMirrorStatus(workerID, mirrorID string, status mirro return status, err } -func (b *boltAdapter) GetMirrorStatus(workerID, mirrorID string) (m mirrorStatus, err error) { +func (b *boltAdapter) GetMirrorStatus(workerID, mirrorID string) (m MirrorStatus, err error) { id := mirrorID + "/" + workerID err = b.db.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(_statusBucketKey)) @@ -130,11 +132,11 @@ func (b *boltAdapter) GetMirrorStatus(workerID, mirrorID string) (m mirrorStatus return } -func (b *boltAdapter) ListMirrorStatus(workerID string) (ms []mirrorStatus, err error) { +func (b *boltAdapter) ListMirrorStatus(workerID string) (ms []MirrorStatus, err error) { err = b.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(_statusBucketKey)) c := bucket.Cursor() - var m mirrorStatus + var m MirrorStatus for k, v := c.First(); k != nil; k, v = c.Next() { if wID := strings.Split(string(k), "/")[1]; wID == workerID { jsonErr := json.Unmarshal(v, &m) @@ -150,11 +152,11 @@ func (b *boltAdapter) ListMirrorStatus(workerID string) (ms []mirrorStatus, err return } -func (b *boltAdapter) ListAllMirrorStatus() (ms []mirrorStatus, err error) { +func (b *boltAdapter) ListAllMirrorStatus() (ms []MirrorStatus, err error) { err = b.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(_statusBucketKey)) c := bucket.Cursor() - var m mirrorStatus + var m MirrorStatus for k, v := c.First(); k != nil; k, v = c.Next() { jsonErr := json.Unmarshal(v, &m) if jsonErr != nil { diff --git a/manager/db_test.go b/manager/db_test.go index 4c01d3e..5cd7456 100644 --- a/manager/db_test.go +++ b/manager/db_test.go @@ -31,7 +31,7 @@ func TestBoltAdapter(t *testing.T) { testWorkerIDs := []string{"test_worker1", "test_worker2"} Convey("create worker", func() { for _, id := range testWorkerIDs { - w := workerStatus{ + w := WorkerStatus{ ID: id, Token: "token_" + id, LastOnline: time.Now(), @@ -58,7 +58,7 @@ func TestBoltAdapter(t *testing.T) { }) Convey("update mirror status", func() { - status1 := mirrorStatus{ + status1 := MirrorStatus{ Name: "arch-sync1", Worker: testWorkerIDs[0], IsMaster: true, @@ -67,7 +67,7 @@ func TestBoltAdapter(t *testing.T) { Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "3GB", } - status2 := mirrorStatus{ + status2 := MirrorStatus{ Name: "arch-sync2", Worker: testWorkerIDs[1], IsMaster: true, @@ -94,7 +94,7 @@ func TestBoltAdapter(t *testing.T) { Convey("list mirror status", func() { ms, err := boltDB.ListMirrorStatus(testWorkerIDs[0]) So(err, ShouldBeNil) - expectedJSON, err := json.Marshal([]mirrorStatus{status1}) + expectedJSON, err := json.Marshal([]MirrorStatus{status1}) So(err, ShouldBeNil) actualJSON, err := json.Marshal(ms) So(err, ShouldBeNil) @@ -104,7 +104,7 @@ func TestBoltAdapter(t *testing.T) { Convey("list all mirror status", func() { ms, err := boltDB.ListAllMirrorStatus() So(err, ShouldBeNil) - expectedJSON, err := json.Marshal([]mirrorStatus{status1, status2}) + expectedJSON, err := json.Marshal([]MirrorStatus{status1, status2}) So(err, ShouldBeNil) actualJSON, err := json.Marshal(ms) So(err, ShouldBeNil) diff --git a/manager/server.go b/manager/server.go index 5102aae..b2ab87b 100644 --- a/manager/server.go +++ b/manager/server.go @@ -30,12 +30,19 @@ func (s *managerServer) listAllJobs(c *gin.Context) { s.returnErrJSON(c, http.StatusInternalServerError, err) return } - c.JSON(http.StatusOK, mirrorStatusList) + webMirStatusList := []webMirrorStatus{} + for _, m := range mirrorStatusList { + webMirStatusList = append( + webMirStatusList, + convertMirrorStatus(m), + ) + } + c.JSON(http.StatusOK, webMirStatusList) } // listWrokers respond with informations of all the workers func (s *managerServer) listWorkers(c *gin.Context) { - var workerInfos []WorkerInfoMsg + var workerInfos []WorkerStatus workers, err := s.adapter.ListWorkers() if err != nil { err := fmt.Errorf("failed to list workers: %s", @@ -47,7 +54,7 @@ func (s *managerServer) listWorkers(c *gin.Context) { } for _, w := range workers { workerInfos = append(workerInfos, - WorkerInfoMsg{ + WorkerStatus{ ID: w.ID, LastOnline: w.LastOnline, }) @@ -57,7 +64,7 @@ func (s *managerServer) listWorkers(c *gin.Context) { // registerWorker register an newly-online worker func (s *managerServer) registerWorker(c *gin.Context) { - var _worker workerStatus + var _worker WorkerStatus c.BindJSON(&_worker) newWorker, err := s.adapter.CreateWorker(_worker) if err != nil { @@ -95,7 +102,7 @@ func (s *managerServer) returnErrJSON(c *gin.Context, code int, err error) { func (s *managerServer) updateJobOfWorker(c *gin.Context) { workerID := c.Param("id") - var status mirrorStatus + var status MirrorStatus c.BindJSON(&status) mirrorName := status.Name newStatus, err := s.adapter.UpdateMirrorStatus(workerID, mirrorName, status) diff --git a/manager/server_test.go b/manager/server_test.go index 6a14d9b..beeb80b 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -26,11 +26,11 @@ func TestHTTPServer(t *testing.T) { s := makeHTTPServer(false) So(s, ShouldNotBeNil) s.setDBAdapter(&mockDBAdapter{ - workerStore: map[string]workerStatus{ - _magicBadWorkerID: workerStatus{ + workerStore: map[string]WorkerStatus{ + _magicBadWorkerID: WorkerStatus{ ID: _magicBadWorkerID, }}, - statusStore: make(map[string]mirrorStatus), + statusStore: make(map[string]MirrorStatus), }) port := rand.Intn(10000) + 20000 baseURL := fmt.Sprintf("http://127.0.0.1:%d", port) @@ -62,7 +62,7 @@ func TestHTTPServer(t *testing.T) { }) Convey("when register a worker", func(ctx C) { - w := workerStatus{ + w := WorkerStatus{ ID: "test_worker1", } resp, err := postJSON(baseURL+"/workers", w) @@ -74,14 +74,14 @@ func TestHTTPServer(t *testing.T) { resp, err := http.Get(baseURL + "/workers") So(err, ShouldBeNil) defer resp.Body.Close() - var actualResponseObj []WorkerInfoMsg + var actualResponseObj []WorkerStatus err = json.NewDecoder(resp.Body).Decode(&actualResponseObj) So(err, ShouldBeNil) So(len(actualResponseObj), ShouldEqual, 2) }) Convey("update mirror status of a existed worker", func(ctx C) { - status := mirrorStatus{ + status := MirrorStatus{ Name: "arch-sync1", Worker: "test_worker1", IsMaster: true, @@ -97,7 +97,7 @@ func TestHTTPServer(t *testing.T) { Convey("list mirror status of an existed worker", func(ctx C) { - expectedResponse, err := json.Marshal([]mirrorStatus{status}) + expectedResponse, err := json.Marshal([]MirrorStatus{status}) So(err, ShouldBeNil) resp, err := http.Get(baseURL + "/workers/test_worker1/jobs") So(err, ShouldBeNil) @@ -110,7 +110,9 @@ func TestHTTPServer(t *testing.T) { }) Convey("list all job status of all workers", func(ctx C) { - expectedResponse, err := json.Marshal([]mirrorStatus{status}) + expectedResponse, err := json.Marshal( + []webMirrorStatus{convertMirrorStatus(status)}, + ) So(err, ShouldBeNil) resp, err := http.Get(baseURL + "/jobs") So(err, ShouldBeNil) @@ -125,7 +127,7 @@ func TestHTTPServer(t *testing.T) { Convey("update mirror status of an inexisted worker", func(ctx C) { invalidWorker := "test_worker2" - status := mirrorStatus{ + status := MirrorStatus{ Name: "arch-sync2", Worker: invalidWorker, IsMaster: true, @@ -150,7 +152,7 @@ func TestHTTPServer(t *testing.T) { workerPort := rand.Intn(10000) + 30000 bindAddress := fmt.Sprintf("127.0.0.1:%d", workerPort) workerBaseURL := fmt.Sprintf("http://%s", bindAddress) - w := workerStatus{ + w := WorkerStatus{ ID: "test_worker_cmd", URL: workerBaseURL + "/cmd", } @@ -208,16 +210,16 @@ func TestHTTPServer(t *testing.T) { } type mockDBAdapter struct { - workerStore map[string]workerStatus - statusStore map[string]mirrorStatus + workerStore map[string]WorkerStatus + statusStore map[string]MirrorStatus } func (b *mockDBAdapter) Init() error { return nil } -func (b *mockDBAdapter) ListWorkers() ([]workerStatus, error) { - workers := make([]workerStatus, len(b.workerStore)) +func (b *mockDBAdapter) ListWorkers() ([]WorkerStatus, error) { + workers := make([]WorkerStatus, len(b.workerStore)) idx := 0 for _, w := range b.workerStore { workers[idx] = w @@ -226,15 +228,15 @@ func (b *mockDBAdapter) ListWorkers() ([]workerStatus, error) { return workers, nil } -func (b *mockDBAdapter) GetWorker(workerID string) (workerStatus, error) { +func (b *mockDBAdapter) GetWorker(workerID string) (WorkerStatus, error) { w, ok := b.workerStore[workerID] if !ok { - return workerStatus{}, fmt.Errorf("invalid workerId") + return WorkerStatus{}, fmt.Errorf("invalid workerId") } return w, nil } -func (b *mockDBAdapter) CreateWorker(w workerStatus) (workerStatus, error) { +func (b *mockDBAdapter) CreateWorker(w WorkerStatus) (WorkerStatus, error) { // _, ok := b.workerStore[w.ID] // if ok { // return workerStatus{}, fmt.Errorf("duplicate worker name") @@ -243,19 +245,19 @@ func (b *mockDBAdapter) CreateWorker(w workerStatus) (workerStatus, error) { return w, nil } -func (b *mockDBAdapter) GetMirrorStatus(workerID, mirrorID string) (mirrorStatus, error) { +func (b *mockDBAdapter) GetMirrorStatus(workerID, mirrorID string) (MirrorStatus, error) { id := mirrorID + "/" + workerID status, ok := b.statusStore[id] if !ok { - return mirrorStatus{}, fmt.Errorf("no mirror %s exists in worker %s", mirrorID, workerID) + return MirrorStatus{}, fmt.Errorf("no mirror %s exists in worker %s", mirrorID, workerID) } return status, nil } -func (b *mockDBAdapter) UpdateMirrorStatus(workerID, mirrorID string, status mirrorStatus) (mirrorStatus, error) { +func (b *mockDBAdapter) UpdateMirrorStatus(workerID, mirrorID string, status MirrorStatus) (MirrorStatus, error) { // if _, ok := b.workerStore[workerID]; !ok { // // unregistered worker - // return mirrorStatus{}, fmt.Errorf("invalid workerID %s", workerID) + // return MirrorStatus{}, fmt.Errorf("invalid workerID %s", workerID) // } id := mirrorID + "/" + workerID @@ -263,11 +265,11 @@ func (b *mockDBAdapter) UpdateMirrorStatus(workerID, mirrorID string, status mir return status, nil } -func (b *mockDBAdapter) ListMirrorStatus(workerID string) ([]mirrorStatus, error) { - var mirrorStatusList []mirrorStatus +func (b *mockDBAdapter) ListMirrorStatus(workerID string) ([]MirrorStatus, error) { + var mirrorStatusList []MirrorStatus // simulating a database fail if workerID == _magicBadWorkerID { - return []mirrorStatus{}, fmt.Errorf("database fail") + return []MirrorStatus{}, fmt.Errorf("database fail") } for k, v := range b.statusStore { if wID := strings.Split(k, "/")[1]; wID == workerID { @@ -277,8 +279,8 @@ func (b *mockDBAdapter) ListMirrorStatus(workerID string) ([]mirrorStatus, error return mirrorStatusList, nil } -func (b *mockDBAdapter) ListAllMirrorStatus() ([]mirrorStatus, error) { - var mirrorStatusList []mirrorStatus +func (b *mockDBAdapter) ListAllMirrorStatus() ([]MirrorStatus, error) { + var mirrorStatusList []MirrorStatus for _, v := range b.statusStore { mirrorStatusList = append(mirrorStatusList, v) } diff --git a/manager/status.go b/manager/status.go index 8c6150c..8df2b3c 100644 --- a/manager/status.go +++ b/manager/status.go @@ -2,114 +2,61 @@ package manager import ( "encoding/json" - "errors" - "fmt" "strconv" "time" . "github.com/tuna/tunasync/internal" ) -type mirrorStatus struct { - Name string - Worker string - IsMaster bool - Status SyncStatus - LastUpdate time.Time - Upstream string - Size string // approximate size +type textTime struct { + time.Time } -func (s mirrorStatus) MarshalJSON() ([]byte, error) { - m := map[string]interface{}{ - "name": s.Name, - "worker": s.Worker, - "is_master": s.IsMaster, - "status": s.Status, - "last_update": s.LastUpdate.Format("2006-01-02 15:04:05"), - "last_update_ts": fmt.Sprintf("%d", s.LastUpdate.Unix()), - "size": s.Size, - "upstream": s.Upstream, - } - return json.Marshal(m) +func (t textTime) MarshalJSON() ([]byte, error) { + return json.Marshal(t.Format("2006-01-02 15:04:05")) +} +func (t *textTime) UnmarshalJSON(b []byte) error { + s := string(b) + t2, err := time.ParseInLocation(`"2006-01-02 15:04:05"`, s, time.Local) + *t = textTime{t2} + return err } -func (s *mirrorStatus) UnmarshalJSON(v []byte) error { - var m map[string]interface{} +type stampTime struct { + time.Time +} - err := json.Unmarshal(v, &m) +func (t stampTime) MarshalJSON() ([]byte, error) { + return json.Marshal(t.Unix()) +} +func (t *stampTime) UnmarshalJSON(b []byte) error { + ts, err := strconv.Atoi(string(b)) if err != nil { return err } - - if name, ok := m["name"]; ok { - if s.Name, ok = name.(string); !ok { - return errors.New("name should be a string") - } - } else { - return errors.New("key `name` does not exist in the json") - } - if isMaster, ok := m["is_master"]; ok { - if s.IsMaster, ok = isMaster.(bool); !ok { - return errors.New("is_master should be a string") - } - } else { - return errors.New("key `is_master` does not exist in the json") - } - if _worker, ok := m["worker"]; ok { - if s.Worker, ok = _worker.(string); !ok { - return errors.New("worker should be a string") - } - } else { - return errors.New("key `worker` does not exist in the json") - } - if upstream, ok := m["upstream"]; ok { - if s.Upstream, ok = upstream.(string); !ok { - return errors.New("upstream should be a string") - } - } else { - return errors.New("key `upstream` does not exist in the json") - } - if size, ok := m["size"]; ok { - if s.Size, ok = size.(string); !ok { - return errors.New("size should be a string") - } - } else { - return errors.New("key `size` does not exist in the json") - } - // tricky: status - if status, ok := m["status"]; ok { - if ss, ok := status.(string); ok { - err := json.Unmarshal([]byte(`"`+ss+`"`), &(s.Status)) - if err != nil { - return err - } - } else { - return errors.New("status should be a string") - } - } else { - return errors.New("key `status` does not exist in the json") - } - // tricky: last update - if lastUpdate, ok := m["last_update_ts"]; ok { - if sts, ok := lastUpdate.(string); ok { - ts, err := strconv.Atoi(sts) - if err != nil { - return fmt.Errorf("last_update_ts should be a interger, got: %s", sts) - } - s.LastUpdate = time.Unix(int64(ts), 0) - } else { - return fmt.Errorf("last_update_ts should be a string of integer, got: %s", lastUpdate) - } - } else { - return errors.New("key `last_update_ts` does not exist in the json") - } - return nil + *t = stampTime{time.Unix(int64(ts), 0)} + return err } -type workerStatus struct { - ID string `json:"id"` // worker name - Token string `json:"token"` // session token - URL string `json:"url"` // worker url - LastOnline time.Time `json:"last_online"` // last seen +// webMirrorStatus is the mirror status to be shown in the web page +type webMirrorStatus struct { + Name string `json:"name"` + IsMaster bool `json:"is_master"` + Status SyncStatus `json:"status"` + LastUpdate textTime `json:"last_update"` + LastUpdateTs stampTime `json:"last_update_ts"` + Upstream string `json:"upstream"` + Size string `json:"size"` // approximate size +} + +func convertMirrorStatus(m MirrorStatus) webMirrorStatus { + return webMirrorStatus{ + Name: m.Name, + IsMaster: m.IsMaster, + Status: m.Status, + LastUpdate: textTime{m.LastUpdate}, + LastUpdateTs: stampTime{m.LastUpdate}, + Upstream: m.Upstream, + Size: m.Size, + } } diff --git a/manager/status_test.go b/manager/status_test.go index 06260d9..2ca0f1e 100644 --- a/manager/status_test.go +++ b/manager/status_test.go @@ -15,26 +15,29 @@ func TestStatus(t *testing.T) { tz := "Asia/Shanghai" loc, err := time.LoadLocation(tz) So(err, ShouldBeNil) - - m := mirrorStatus{ - Name: "tunalinux", - Status: tunasync.Success, - LastUpdate: time.Date(2016, time.April, 16, 23, 8, 10, 0, loc), - Size: "5GB", - Upstream: "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/", + t := time.Date(2016, time.April, 16, 23, 8, 10, 0, loc) + m := webMirrorStatus{ + Name: "tunalinux", + Status: tunasync.Success, + LastUpdate: textTime{t}, + LastUpdateTs: stampTime{t}, + Size: "5GB", + Upstream: "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/", } b, err := json.Marshal(m) So(err, ShouldBeNil) // fmt.Println(string(b)) - var m2 mirrorStatus + var m2 webMirrorStatus err = json.Unmarshal(b, &m2) So(err, ShouldBeNil) // fmt.Printf("%#v", m2) So(m2.Name, ShouldEqual, m.Name) So(m2.Status, ShouldEqual, m.Status) So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix()) + So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix()) So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) + So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) So(m2.Size, ShouldEqual, m.Size) So(m2.Upstream, ShouldEqual, m.Upstream) }) diff --git a/worker/worker.go b/worker/worker.go index 063f932..02ed1f2 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -291,7 +291,7 @@ func (w *Worker) registorWorker() { w.cfg.Manager.APIBase, ) - msg := WorkerInfoMsg{ + msg := WorkerStatus{ ID: w.Name(), URL: w.URL(), } From f8151e689fe6924358b1e569a4f7faab546fb0c3 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 19:37:29 +0800 Subject: [PATCH 43/66] refactor(worker): export worker's LoadConfig --- worker/config.go | 3 ++- worker/config_test.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/worker/config.go b/worker/config.go index f766ee1..bb3bd12 100644 --- a/worker/config.go +++ b/worker/config.go @@ -76,7 +76,8 @@ type mirrorConfig struct { Stage1Profile string `toml:"stage1_profile"` } -func loadConfig(cfgFile string) (*Config, error) { +// LoadConfig loads configuration +func LoadConfig(cfgFile string) (*Config, error) { if _, err := os.Stat(cfgFile); err != nil { return nil, err } diff --git a/worker/config_test.go b/worker/config_test.go index 94d8bae..dbebcb7 100644 --- a/worker/config_test.go +++ b/worker/config_test.go @@ -54,7 +54,7 @@ exclude_file = "/etc/tunasync.d/fedora-exclude.txt" ` Convey("When giving invalid file", t, func() { - cfg, err := loadConfig("/path/to/invalid/file") + cfg, err := LoadConfig("/path/to/invalid/file") So(err, ShouldNotBeNil) So(cfg, ShouldBeNil) }) @@ -68,7 +68,7 @@ exclude_file = "/etc/tunasync.d/fedora-exclude.txt" So(err, ShouldEqual, nil) defer tmpfile.Close() - cfg, err := loadConfig(tmpfile.Name()) + cfg, err := LoadConfig(tmpfile.Name()) So(err, ShouldBeNil) So(cfg.Global.Name, ShouldEqual, "test_worker") So(cfg.Global.Interval, ShouldEqual, 240) @@ -107,7 +107,7 @@ exclude_file = "/etc/tunasync.d/fedora-exclude.txt" So(err, ShouldEqual, nil) defer tmpfile.Close() - cfg, err := loadConfig(tmpfile.Name()) + cfg, err := LoadConfig(tmpfile.Name()) So(err, ShouldBeNil) w := &Worker{ From 84b7bdd713cd76f85b331c97df547b78d758d90c Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 19:37:46 +0800 Subject: [PATCH 44/66] tests: self-signed TLS certificates --- tests/httpClient.go | 16 ++++++++++++++++ tests/httpServer.go | 15 +++++++++++++++ tests/manager.crt | 22 ++++++++++++++++++++++ tests/manager.csr | 18 ++++++++++++++++++ tests/manager.key | 27 +++++++++++++++++++++++++++ tests/req.cnf | 26 ++++++++++++++++++++++++++ tests/rootCA.crt | 23 +++++++++++++++++++++++ tests/rootCA.key | 27 +++++++++++++++++++++++++++ tests/rootCA.srl | 1 + 9 files changed, 175 insertions(+) create mode 100644 tests/httpClient.go create mode 100644 tests/httpServer.go create mode 100644 tests/manager.crt create mode 100644 tests/manager.csr create mode 100644 tests/manager.key create mode 100644 tests/req.cnf create mode 100644 tests/rootCA.crt create mode 100644 tests/rootCA.key create mode 100644 tests/rootCA.srl diff --git a/tests/httpClient.go b/tests/httpClient.go new file mode 100644 index 0000000..a36fd85 --- /dev/null +++ b/tests/httpClient.go @@ -0,0 +1,16 @@ +package main + +import ( + "fmt" + + "github.com/tuna/tunasync/internal" +) + +func main() { + cfg, err := internal.GetTLSConfig("rootCA.crt") + fmt.Println(err) + var msg map[string]string + resp, err := internal.GetJSON("https://localhost:5002/", &msg, cfg) + fmt.Println(err) + fmt.Println(resp) +} diff --git a/tests/httpServer.go b/tests/httpServer.go new file mode 100644 index 0000000..822eb56 --- /dev/null +++ b/tests/httpServer.go @@ -0,0 +1,15 @@ +package main + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +func main() { + s := gin.Default() + s.GET("/", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"msg": "passed"}) + }) + s.RunTLS(":5002", "manager.crt", "manager.key") +} diff --git a/tests/manager.crt b/tests/manager.crt new file mode 100644 index 0000000..5e24903 --- /dev/null +++ b/tests/manager.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDmjCCAoKgAwIBAgIJANsBsjPEVQ3CMA0GCSqGSIb3DQEBCwUAMIGEMQswCQYD +VQQGEwJDTjELMAkGA1UECAwCQkoxETAPBgNVBAcMCFRzaW5naHVhMQ0wCwYDVQQK +DARUVU5BMRAwDgYDVQQLDAdNaXJyb3JzMRIwEAYDVQQDDAlsb2NhbGhvc3QxIDAe +BgkqhkiG9w0BCQEWEXJvb3RAbWlycm9ycy50dW5hMB4XDTE2MDQyODExMzAwNloX +DTI2MDQyNjExMzAwNlowTzELMAkGA1UEBhMCQ04xCzAJBgNVBAgMAkJKMRAwDgYD +VQQHDAdCZWlqaW5nMQ0wCwYDVQQLDARUVU5BMRIwEAYDVQQDDAlsb2NhbGhvc3Qw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDsQ2iLGiyJgMe1Y4kvmkZ8 +2fGOCZWp1rYZ5DWCqKZ4WtlmnxHYT4ZkopCCNo0FoQZ4TmDPWJctfRcHaTbidtFp +u416rg9zcg9jlwtO0OKNTzS0RkiF2zUyX4bGFx85xu9z18JYwnWej4fvpfGsPUev +T/roLkuUyaHJc+LeOIT0e9+mwSUC6KckGC86B5PK1gyFFjnuNeuk9TL6jnzAcczZ +sCF8gzDAtxEN++fQFxY/ZMnyAGzmyo9qVqJwLB7ANU6PfcIpcaD0GRDqOFRyDwCM +WmLHIZAltmDOKpd1Qj0N4nsPbsExQHBP01B2iB18CR8zG2DrCi77ZafNvQjL7KZX +AgMBAAGjQzBBMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgXgMCcGA1UdEQQgMB6CCWxv +Y2FsaG9zdIIRbWFuYWdlci5sb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggEBAKrN +zOxDqtZzx8Lj+0/EahuINCrJWWA29jnbz7u4nJ+38zLW4WFJLF6DWSaFOLjQjwUk +X8RD/Ja5UW1eK0Ur+Q9pkNxpqZstOBHs/SuudMwfYu48uMs938+sS58DMV3Yeyjx +Jk8RaWgWrsrTXBpxmGbjWSV+HCoM56lzOSVp1g5H0ksbYakxR6lmkFagptcC2HEL +QMtgnQc+DPXUMGkAGaWOx7Wrwby2elDPafP1eZEBR+tBdkD4C2/bDAdK2soEN48K +EdWYFiWiefGb+Vf60mrud+dRF069nOKYOg6xTDg3jy4PIJp44Luxn7vOZRV/zmfT +0BZ5A+Zy/iAtg7hw5sE= +-----END CERTIFICATE----- diff --git a/tests/manager.csr b/tests/manager.csr new file mode 100644 index 0000000..c09987c --- /dev/null +++ b/tests/manager.csr @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC5jCCAc4CAQAwTzELMAkGA1UEBhMCQ04xCzAJBgNVBAgMAkJKMRAwDgYDVQQH +DAdCZWlqaW5nMQ0wCwYDVQQLDARUVU5BMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDsQ2iLGiyJgMe1Y4kvmkZ82fGO +CZWp1rYZ5DWCqKZ4WtlmnxHYT4ZkopCCNo0FoQZ4TmDPWJctfRcHaTbidtFpu416 +rg9zcg9jlwtO0OKNTzS0RkiF2zUyX4bGFx85xu9z18JYwnWej4fvpfGsPUevT/ro +LkuUyaHJc+LeOIT0e9+mwSUC6KckGC86B5PK1gyFFjnuNeuk9TL6jnzAcczZsCF8 +gzDAtxEN++fQFxY/ZMnyAGzmyo9qVqJwLB7ANU6PfcIpcaD0GRDqOFRyDwCMWmLH +IZAltmDOKpd1Qj0N4nsPbsExQHBP01B2iB18CR8zG2DrCi77ZafNvQjL7KZXAgMB +AAGgUjBQBgkqhkiG9w0BCQ4xQzBBMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgXgMCcG +A1UdEQQgMB6CCWxvY2FsaG9zdIIRbWFuYWdlci5sb2NhbGhvc3QwDQYJKoZIhvcN +AQELBQADggEBAOsVix8POTWeY1uGRSatGX8D9UKZxIGsquOMOWyucSUqEnkGmTri +ketJKcKXuRP3bHsHM+XGbVm0qisfCqg5p1MX0P2yw87+zqAVXSHEuuYLeD75qnu+ +yraydJh6NDp9cwHQxAvFK2Dav8OXHEaug00ZZ3U/Mt2q/b6b2d3ihtGU+wU2Yl4b +xBMIcqsVHapKJOQd+MJBaP2GojCwLE1yuI5Wg6iffgsydoAt+51CPUDs9/KRypqm +zlEPmljToZBl/y/TvUBA1egAnnkXMWzhvK75GFRSPizPRUsqSfu7qysYKcTUseqd +RBP67pHi9Hhmi4rRvytXtFF3ju/MtJ/+wxk= +-----END CERTIFICATE REQUEST----- diff --git a/tests/manager.key b/tests/manager.key new file mode 100644 index 0000000..c88b0ad --- /dev/null +++ b/tests/manager.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEA7ENoixosiYDHtWOJL5pGfNnxjgmVqda2GeQ1gqimeFrZZp8R +2E+GZKKQgjaNBaEGeE5gz1iXLX0XB2k24nbRabuNeq4Pc3IPY5cLTtDijU80tEZI +hds1Ml+GxhcfOcbvc9fCWMJ1no+H76XxrD1Hr0/66C5LlMmhyXPi3jiE9HvfpsEl +AuinJBgvOgeTytYMhRY57jXrpPUy+o58wHHM2bAhfIMwwLcRDfvn0BcWP2TJ8gBs +5sqPalaicCwewDVOj33CKXGg9BkQ6jhUcg8AjFpixyGQJbZgziqXdUI9DeJ7D27B +MUBwT9NQdogdfAkfMxtg6wou+2Wnzb0Iy+ymVwIDAQABAoIBAQC1Vy/gxKA2kg+3 +G8TqMqGzppyPBrBHAbQXv1+K/+N2MVT4PVO3EaL3jwcXysMG9QdAQ/hubXLryT1p +xMoJnGUzoG8BIKRfWcaSDBbz0cRx7b9oNyHnC8+S8FtDo++lqxmTcqGK+wbIQyZ1 +PIt4RjjFSMAugYolk3WIaFhTdFIoS4ozk/VZNyYzWg2XEjMugL9Pe/zU0vlzQPRj +4vUhmX4lvuJ1/T3XR53vMU1cMiwxSGbLeG4F4zshzIh9LfbHFKNweO/YIfmFJVaS +C7aYl9Jss5SDviUuowHcgqk6oivWr3cxiVma/zc5SMeWzgmGcDX6izQx1Y8PPsUy +vsuLHGZRAoGBAP2DDKVc3FSslIiqV/8iHKh4sRPEJ6j03il62LwzRBmmZb3t6eD1 +oxAxJA+3dEcjxzOEdPng6Vtvbd5BqFy5kRTkqjWA03HjsFGgItbhzfw3CtsSH1R1 +IlxvA71+k65yP0QY9xwYWUBXNQtp0cLT1hlDwv+W5UCC1lxtDpyHlsBNAoGBAO6V +BZDawpohmzLtc5O4FXyt5B/hR79VNs5bfOj856xNnf6FREVgxCgoZvYlUh80lzSN +SQl68llCQJCWlndcdafnu5PRo2WiuJbIMcNdwZY6wT+gT/twXwE6nk7RDg9KaARc +OCKjLJLATOslF38K9n1I0Y/ZdCBFNcBxfHHlaTMzAoGBANQ+5NaJsXo+5ziojXXw +xFeUfITVBHNjV6EY1d5zeX+UHbhvORF79mK3Eb8K1BI/dSa/rgQK9rTzzON4yxGe +10XL0GltCxpeC5+7V4/ai0+vcapKOOrICtWiqFn9YH1771X/JNxj0k2Y9bMxjEn2 +e1i5r8e3OQbSw8+sCsCokGE9AoGBAMx4rT97LQL5wFBCTyaPwuKLCZME+P+S4Ziz +sfbgIRF7p+elgWBQUWz1S2CzlZEm+lvQpoLYevFipYEFfkkn1bIkGY/TQE1vyvF2 ++6crKCk/i7WjCEk/Aj1EZr63zmvuYf0yp+2PmTjgVEvHCz8XPy8ahHfbbvnlNu8K +lBPtAF8fAoGAXuW/i9hu4sgIflWHN+QPN1je4QVMB/Ej8IGMqT9Dde0aCf95OqFp +yct1Oz8R2VLsKI1pxIqIBrnCogHKVkYAYlnRxcykWwy2uhQrDK6CPVmgXg3Yv+7S +kbXHpBlfVFInugn3T+Hvn1uYJ5Ih7OIfcCwZ+6B2Zal7O4RhELuk4rM= +-----END RSA PRIVATE KEY----- diff --git a/tests/req.cnf b/tests/req.cnf new file mode 100644 index 0000000..246fe72 --- /dev/null +++ b/tests/req.cnf @@ -0,0 +1,26 @@ +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req + +[req_distinguished_name] +countryName = Country Name (2 letter code) +countryName_default = CN +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = BJ +localityName = Locality Name (eg, city) +localityName_default = Beijing +organizationalUnitName = Organizational Unit Name (eg, section) +organizationalUnitName_default = TUNA +commonName = Common Name (server FQDN or domain name) +commonName_default = localhost +commonName_max = 64 + +[v3_req] +# Extensions to add to a certificate request +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = localhost +DNS.2 = manager.localhost diff --git a/tests/rootCA.crt b/tests/rootCA.crt new file mode 100644 index 0000000..f53226a --- /dev/null +++ b/tests/rootCA.crt @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID3TCCAsWgAwIBAgIJAJ1h7cAbsEpbMA0GCSqGSIb3DQEBCwUAMIGEMQswCQYD +VQQGEwJDTjELMAkGA1UECAwCQkoxETAPBgNVBAcMCFRzaW5naHVhMQ0wCwYDVQQK +DARUVU5BMRAwDgYDVQQLDAdNaXJyb3JzMRIwEAYDVQQDDAlsb2NhbGhvc3QxIDAe +BgkqhkiG9w0BCQEWEXJvb3RAbWlycm9ycy50dW5hMB4XDTE2MDQyODExMjcxNloX +DTI2MDQyNjExMjcxNlowgYQxCzAJBgNVBAYTAkNOMQswCQYDVQQIDAJCSjERMA8G +A1UEBwwIVHNpbmdodWExDTALBgNVBAoMBFRVTkExEDAOBgNVBAsMB01pcnJvcnMx +EjAQBgNVBAMMCWxvY2FsaG9zdDEgMB4GCSqGSIb3DQEJARYRcm9vdEBtaXJyb3Jz +LnR1bmEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDGPtjiqI89E/mz +3JuWvqbwihQczDug9GiyP5axNT+WkJka0qL+U09V05cn6qXX/JK0BHxqSPYEZy3R +hkLIrtR0LPSk8RCxU9mv11FRigl5NevWbbzJkM2aBS1KIpD07Kk+UJkp/dsIWeNq +Mo/4edkLqob+gIG5IQM/B1mPuAVUrqAVGRAlA1qXv2ahWcdZrbybMrQ9nBPbTwcg +qbK6ytJ2K8GpuWdr+72SJXxIN0rmBfyHQuHwpRMP6XzTCEYd0TCr6YQ+tWnrpk8c +djFKVjIwg22jHUcmVYXNxRw66JPK2aZrL3RkRmlJoIhd5np+SbRkWmbS5zNTgKc8 +TKUskCCVAgMBAAGjUDBOMB0GA1UdDgQWBBS6lED67P/J7snFaxZcdr0gSE/oZDAf +BgNVHSMEGDAWgBS6lED67P/J7snFaxZcdr0gSE/oZDAMBgNVHRMEBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4IBAQCh9mwuLSnDBoIxF5XsFnv4lrNvlGvyRffDa9/wh7Pb +s9rBKfKPO+8Yy7H57Os4Dl/2QoQTjMsvFJTY1TKE3zTDxPAaM5xmgxv3DHFFSG8r +G9zEKyDAVzsdu1kSXvJLIdaycSXCWUjRIiYI153N5TUGtq6lctPeOv/w0P6S8KXP +VgBpiJWiexUOYXVin2zrkbSRkNVntDEbDr5cQ0RznpyqAfKt990VzUjORarh0zyb ++FG9pX/gjO8atGhIuA7hqxUwy4Ov70SxeiiK+POgp/Km9y36G7KM+KZKsj+8JQIq +6/it/KzzDE/awOSw2Ti0ZqCMUCIrsDOA9nmc+t0bERON +-----END CERTIFICATE----- diff --git a/tests/rootCA.key b/tests/rootCA.key new file mode 100644 index 0000000..0662e94 --- /dev/null +++ b/tests/rootCA.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAxj7Y4qiPPRP5s9yblr6m8IoUHMw7oPRosj+WsTU/lpCZGtKi +/lNPVdOXJ+ql1/yStAR8akj2BGct0YZCyK7UdCz0pPEQsVPZr9dRUYoJeTXr1m28 +yZDNmgUtSiKQ9OypPlCZKf3bCFnjajKP+HnZC6qG/oCBuSEDPwdZj7gFVK6gFRkQ +JQNal79moVnHWa28mzK0PZwT208HIKmyusrSdivBqblna/u9kiV8SDdK5gX8h0Lh +8KUTD+l80whGHdEwq+mEPrVp66ZPHHYxSlYyMINtox1HJlWFzcUcOuiTytmmay90 +ZEZpSaCIXeZ6fkm0ZFpm0uczU4CnPEylLJAglQIDAQABAoIBAEkIvj5CewK1aTip +/Wf7tOTI+b/iPdV+NVP1uT5vr414l+8ZypKHfqLP4NAD0jVQB3vqayt81aWpnWej +XtcwEXT7WuWpKc0qZvgxCvRPNk5BXzEQHIzlm9kyLw0wztZsma0rZEHkE91vwChP +mFqiCSQTHsiD70aUsu11d7lKwiv/ww0pty8OmItgL7eefq6UeIidymYSJN6j7OHJ ++Wp6PSKeYJ2/hSVx/F6upGMBJxjaNs9Q53IgH7YwrPThjyVnpyavbJEcawdpdhNo +Y7XqnLYKQiHi86L2Rr7C7g7cv+89GcApweNhDaJUlGzOLnN+3++7n91+S0yjI4CW +/WCY4gECgYEA73z3yzkZ4gk+36g49ZyR+es1VYDCXrRGIpEZTneDK87h9wPCYi9b +5/tvCRfWzJemkhORfE+t8VnC1Ar/VFQJ7gJQXZth/dDecdPQ87pE0fu95BBwQrjG +bRgL+IIloWYh+WhIPVFyLP29lJ6s/gqR0ySKX80NjkHIxnzlNxFgqR0CgYEA0+nv +WK1rgsyrq4jW9Iw3VnuATpSCu0BjiGGEOk/2/LLfN8YG7870o7R0QSAIKz3nI3AM +bTsYiHOlA6d6ZZWfxMz8MPsb0YOTeDTQFg10wxq90Qk02O9nopS1cOSWAK+70lzz +EZyNezNDlI8vsmHu+rYa2MgeFvUQbt+yGNywM9kCgYBHr294vEncGApi5jbOhiRH +27jmNBl6QZnwxN//VdTEqzOGPfDUdPqcsO1mmmUobohkl0joHe2iHc0srXIKKvGh +9b1al6U4VWoQRmf4XJw3ApSvjKAdyLNUemsy4roi2rB2uFlPSW7UusshjnGXxVAr +FHf6/yT8nQJdL4to9WGqnQKBgEEzRNT/5ohD+L26SIjNa2lMblm/D8oVMYqQlmJq +oA936X37i77U6ihEKVCwTlMfpLIek3Q4LoAtNKQ/L0V6F8IxX5aibBi2ZvUhKrTe +RwKQg76BGqV0Y2p+XqTxb8WeTCeZOaA9jrpNN4nJ1F8KCsFQrknsqHVfyUKTyPQl +UoFhAoGBAMXcOnMKhBwhUYZ7pkkntT6vKMBMLz4K2j0mjiYKgoriPn6H4/T2mP13 +qU8VInHwoMN/RIGTCDK2+UUnZfK+aXPhYMUEtFxWQxaWpZ2UopFYCcgYC3yLaBGu +8eWr2G48pJrv/dBxP1nVsgEedfYfjZvyGOrbcRakfiCZOcNHaPb1 +-----END RSA PRIVATE KEY----- diff --git a/tests/rootCA.srl b/tests/rootCA.srl new file mode 100644 index 0000000..7e92478 --- /dev/null +++ b/tests/rootCA.srl @@ -0,0 +1 @@ +DB01B233C4550DC2 From 9865f282590cbb9ce68f974a629f7e4a2514dd9c Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 19:56:45 +0800 Subject: [PATCH 45/66] refactor: manager server --- manager/middleware.go | 2 +- manager/server.go | 148 +++++++++++++++++++++++++++-------------- manager/server_test.go | 4 +- 3 files changed, 100 insertions(+), 54 deletions(-) diff --git a/manager/middleware.go b/manager/middleware.go index 3c2d1ea..67e9266 100644 --- a/manager/middleware.go +++ b/manager/middleware.go @@ -20,7 +20,7 @@ func contextErrorLogger(c *gin.Context) { c.Next() } -func (s *managerServer) workerIDValidator(c *gin.Context) { +func (s *Manager) workerIDValidator(c *gin.Context) { workerID := c.Param("id") _, err := s.adapter.GetWorker(workerID) if err != nil { diff --git a/manager/server.go b/manager/server.go index b2ab87b..8a38cfc 100644 --- a/manager/server.go +++ b/manager/server.go @@ -1,6 +1,7 @@ package manager import ( + "crypto/tls" "fmt" "net/http" @@ -14,13 +15,99 @@ const ( _infoKey = "message" ) -type managerServer struct { - *gin.Engine - adapter dbAdapter +var manager *Manager + +// A Manager represents a manager server +type Manager struct { + cfg *Config + engine *gin.Engine + adapter dbAdapter + tlsConfig *tls.Config +} + +// GetTUNASyncManager returns the manager from config +func GetTUNASyncManager(cfg *Config) *Manager { + if manager != nil { + return manager + } + + // create gin engine + if !cfg.Debug { + gin.SetMode(gin.ReleaseMode) + } + s := &Manager{ + cfg: cfg, + engine: gin.Default(), + adapter: nil, + tlsConfig: nil, + } + + if cfg.Files.CACert != "" { + tlsConfig, err := GetTLSConfig(cfg.Files.CACert) + if err != nil { + logger.Error("Error initializing TLS config: %s", err.Error()) + return nil + } + s.tlsConfig = tlsConfig + } + + if cfg.Files.DBFile != "" { + adapter, err := makeDBAdapter(cfg.Files.DBType, cfg.Files.DBFile) + if err != nil { + logger.Error("Error initializing DB adapter: %s", err.Error()) + return nil + } + s.setDBAdapter(adapter) + } + + // common log middleware + s.engine.Use(contextErrorLogger) + + s.engine.GET("/ping", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{_infoKey: "pong"}) + }) + // list jobs, status page + s.engine.GET("/jobs", s.listAllJobs) + + // list workers + s.engine.GET("/workers", s.listWorkers) + // worker online + s.engine.POST("/workers", s.registerWorker) + + // workerID should be valid in this route group + workerValidateGroup := s.engine.Group("/workers", s.workerIDValidator) + // get job list + workerValidateGroup.GET(":id/jobs", s.listJobsOfWorker) + // post job status + workerValidateGroup.POST(":id/jobs/:job", s.updateJobOfWorker) + + // for tunasynctl to post commands + s.engine.POST("/cmd", s.handleClientCmd) + + manager = s + return s +} + +func (s *Manager) setDBAdapter(adapter dbAdapter) { + s.adapter = adapter +} + +// Run runs the manager server forever +func (s *Manager) Run() { + addr := fmt.Sprintf("%s:%d", s.cfg.Server.Addr, s.cfg.Server.Port) + if s.cfg.Server.SSLCert == "" && s.cfg.Server.SSLKey == "" { + if err := s.engine.Run(addr); err != nil { + panic(err) + } + } else { + if err := s.engine.RunTLS(addr, s.cfg.Server.SSLCert, s.cfg.Server.SSLKey); err != nil { + panic(err) + } + } } // listAllJobs repond with all jobs of specified workers -func (s *managerServer) listAllJobs(c *gin.Context) { +func (s *Manager) listAllJobs(c *gin.Context) { mirrorStatusList, err := s.adapter.ListAllMirrorStatus() if err != nil { err := fmt.Errorf("failed to list all mirror status: %s", @@ -41,7 +128,7 @@ func (s *managerServer) listAllJobs(c *gin.Context) { } // listWrokers respond with informations of all the workers -func (s *managerServer) listWorkers(c *gin.Context) { +func (s *Manager) listWorkers(c *gin.Context) { var workerInfos []WorkerStatus workers, err := s.adapter.ListWorkers() if err != nil { @@ -63,7 +150,7 @@ func (s *managerServer) listWorkers(c *gin.Context) { } // registerWorker register an newly-online worker -func (s *managerServer) registerWorker(c *gin.Context) { +func (s *Manager) registerWorker(c *gin.Context) { var _worker WorkerStatus c.BindJSON(&_worker) newWorker, err := s.adapter.CreateWorker(_worker) @@ -80,7 +167,7 @@ func (s *managerServer) registerWorker(c *gin.Context) { } // listJobsOfWorker respond with all the jobs of the specified worker -func (s *managerServer) listJobsOfWorker(c *gin.Context) { +func (s *Manager) listJobsOfWorker(c *gin.Context) { workerID := c.Param("id") mirrorStatusList, err := s.adapter.ListMirrorStatus(workerID) if err != nil { @@ -94,13 +181,13 @@ func (s *managerServer) listJobsOfWorker(c *gin.Context) { c.JSON(http.StatusOK, mirrorStatusList) } -func (s *managerServer) returnErrJSON(c *gin.Context, code int, err error) { +func (s *Manager) returnErrJSON(c *gin.Context, code int, err error) { c.JSON(code, gin.H{ _errorKey: err.Error(), }) } -func (s *managerServer) updateJobOfWorker(c *gin.Context) { +func (s *Manager) updateJobOfWorker(c *gin.Context) { workerID := c.Param("id") var status MirrorStatus c.BindJSON(&status) @@ -117,7 +204,7 @@ func (s *managerServer) updateJobOfWorker(c *gin.Context) { c.JSON(http.StatusOK, newStatus) } -func (s *managerServer) handleClientCmd(c *gin.Context) { +func (s *Manager) handleClientCmd(c *gin.Context) { var clientCmd ClientCmd c.BindJSON(&clientCmd) workerID := clientCmd.WorkerID @@ -153,44 +240,3 @@ func (s *managerServer) handleClientCmd(c *gin.Context) { // TODO: check response for success c.JSON(http.StatusOK, gin.H{_infoKey: "successfully send command to worker " + workerID}) } - -func (s *managerServer) setDBAdapter(adapter dbAdapter) { - s.adapter = adapter -} - -func makeHTTPServer(debug bool) *managerServer { - // create gin engine - if !debug { - gin.SetMode(gin.ReleaseMode) - } - s := &managerServer{ - gin.Default(), - nil, - } - - // common log middleware - s.Use(contextErrorLogger) - - s.GET("/ping", func(c *gin.Context) { - c.JSON(http.StatusOK, gin.H{_infoKey: "pong"}) - }) - // list jobs, status page - s.GET("/jobs", s.listAllJobs) - - // list workers - s.GET("/workers", s.listWorkers) - // worker online - s.POST("/workers", s.registerWorker) - - // workerID should be valid in this route group - workerValidateGroup := s.Group("/workers", s.workerIDValidator) - // get job list - workerValidateGroup.GET(":id/jobs", s.listJobsOfWorker) - // post job status - workerValidateGroup.POST(":id/jobs/:job", s.updateJobOfWorker) - - // for tunasynctl to post commands - s.POST("/cmd", s.handleClientCmd) - - return s -} diff --git a/manager/server_test.go b/manager/server_test.go index beeb80b..11909ca 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -23,7 +23,7 @@ const ( func TestHTTPServer(t *testing.T) { Convey("HTTP server should work", t, func(ctx C) { InitLogger(true, true, false) - s := makeHTTPServer(false) + s := GetTUNASyncManager(&Config{Debug: false}) So(s, ShouldNotBeNil) s.setDBAdapter(&mockDBAdapter{ workerStore: map[string]WorkerStatus{ @@ -35,7 +35,7 @@ func TestHTTPServer(t *testing.T) { port := rand.Intn(10000) + 20000 baseURL := fmt.Sprintf("http://127.0.0.1:%d", port) go func() { - s.Run(fmt.Sprintf("127.0.0.1:%d", port)) + s.engine.Run(fmt.Sprintf("127.0.0.1:%d", port)) }() time.Sleep(50 * time.Microsecond) resp, err := http.Get(baseURL + "/ping") From 9fbb8ab155a63cd690f4c3bb0cf1643877132b1a Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 21:02:39 +0800 Subject: [PATCH 46/66] refactor(tunasync): 1. refactored manager and worker to support TLS transport 2. if mirror_dir is specified from a mirror config, don't add the mirror name --- manager/server.go | 4 +++- manager/server_test.go | 14 +++++++------- manager/util.go | 13 ------------- worker/runner.go | 8 ++++++++ worker/worker.go | 26 ++++++++++++++++++++------ 5 files changed, 38 insertions(+), 27 deletions(-) delete mode 100644 manager/util.go diff --git a/manager/server.go b/manager/server.go index 8a38cfc..94f40ad 100644 --- a/manager/server.go +++ b/manager/server.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "fmt" "net/http" + "time" "github.com/gin-gonic/gin" @@ -153,6 +154,7 @@ func (s *Manager) listWorkers(c *gin.Context) { func (s *Manager) registerWorker(c *gin.Context) { var _worker WorkerStatus c.BindJSON(&_worker) + _worker.LastOnline = time.Now() newWorker, err := s.adapter.CreateWorker(_worker) if err != nil { err := fmt.Errorf("failed to register worker: %s", @@ -230,7 +232,7 @@ func (s *Manager) handleClientCmd(c *gin.Context) { } // post command to worker - _, err = postJSON(workerURL, workerCmd) + _, err = PostJSON(workerURL, workerCmd, s.tlsConfig) if err != nil { err := fmt.Errorf("post command to worker %s(%s) fail: %s", workerID, workerURL, err.Error()) c.Error(err) diff --git a/manager/server_test.go b/manager/server_test.go index 11909ca..1cc9259 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -65,7 +65,7 @@ func TestHTTPServer(t *testing.T) { w := WorkerStatus{ ID: "test_worker1", } - resp, err := postJSON(baseURL+"/workers", w) + resp, err := PostJSON(baseURL+"/workers", w, nil) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) @@ -90,7 +90,7 @@ func TestHTTPServer(t *testing.T) { Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "3GB", } - resp, err := postJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", baseURL, status.Worker, status.Name), status) + resp, err := PostJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", baseURL, status.Worker, status.Name), status, nil) defer resp.Body.Close() So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) @@ -136,8 +136,8 @@ func TestHTTPServer(t *testing.T) { Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "4GB", } - resp, err := postJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", - baseURL, status.Worker, status.Name), status) + resp, err := PostJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", + baseURL, status.Worker, status.Name), status, nil) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusBadRequest) defer resp.Body.Close() @@ -156,7 +156,7 @@ func TestHTTPServer(t *testing.T) { ID: "test_worker_cmd", URL: workerBaseURL + "/cmd", } - resp, err := postJSON(baseURL+"/workers", w) + resp, err := PostJSON(baseURL+"/workers", w, nil) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) @@ -177,7 +177,7 @@ func TestHTTPServer(t *testing.T) { MirrorID: "ubuntu-sync", WorkerID: "not_exist_worker", } - resp, err := postJSON(baseURL+"/cmd", clientCmd) + resp, err := PostJSON(baseURL+"/cmd", clientCmd, nil) defer resp.Body.Close() So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusBadRequest) @@ -190,7 +190,7 @@ func TestHTTPServer(t *testing.T) { WorkerID: w.ID, } - resp, err := postJSON(baseURL+"/cmd", clientCmd) + resp, err := PostJSON(baseURL+"/cmd", clientCmd, nil) defer resp.Body.Close() So(err, ShouldBeNil) diff --git a/manager/util.go b/manager/util.go deleted file mode 100644 index 4174eea..0000000 --- a/manager/util.go +++ /dev/null @@ -1,13 +0,0 @@ -package manager - -import ( - "bytes" - "encoding/json" - "net/http" -) - -func postJSON(url string, obj interface{}) (*http.Response, error) { - b := new(bytes.Buffer) - json.NewEncoder(b).Encode(obj) - return http.Post(url, "application/json; charset=utf-8", b) -} diff --git a/worker/runner.go b/worker/runner.go index aa8519f..49e27bd 100644 --- a/worker/runner.go +++ b/worker/runner.go @@ -37,6 +37,14 @@ func newCmdJob(cmdAndArgs []string, workingDir string, env map[string]string) *c panic("Command length should be at least 1!") } + logger.Debug("Executing command %s at %s", cmdAndArgs[0], workingDir) + if _, err := os.Stat(workingDir); os.IsNotExist(err) { + logger.Debug("Making dir %s", workingDir) + if err = os.MkdirAll(workingDir, 0755); err != nil { + logger.Error("Error making dir %s", workingDir) + } + } + cmd.Dir = workingDir cmd.Env = newEnviron(env, true) diff --git a/worker/worker.go b/worker/worker.go index 02ed1f2..56bffa6 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -49,6 +49,16 @@ func GetTUNASyncWorker(cfg *Config) *Worker { schedule: newScheduleQueue(), mirrorStatus: make(map[string]SyncStatus), } + + if cfg.Manager.CACert != "" { + tlsConfig, err := GetTLSConfig(cfg.Manager.CACert) + if err != nil { + logger.Error("Failed to init TLS config: %s", err.Error()) + return nil + } + w.tlsConfig = tlsConfig + } + w.initJobs() w.makeHTTPServer() tunasyncWorker = w @@ -75,7 +85,9 @@ func (w *Worker) initProviders() { logDir = c.Global.LogDir } if mirrorDir == "" { - mirrorDir = c.Global.MirrorDir + mirrorDir = filepath.Join( + c.Global.MirrorDir, mirror.Name, + ) } logDir = formatLogDir(logDir, mirror) @@ -87,7 +99,7 @@ func (w *Worker) initProviders() { name: mirror.Name, upstreamURL: mirror.Upstream, command: mirror.Command, - workingDir: filepath.Join(mirrorDir, mirror.Name), + workingDir: mirrorDir, logDir: logDir, logFile: filepath.Join(logDir, "latest.log"), interval: time.Duration(mirror.Interval) * time.Minute, @@ -102,9 +114,10 @@ func (w *Worker) initProviders() { rc := rsyncConfig{ name: mirror.Name, upstreamURL: mirror.Upstream, + rsyncCmd: mirror.Command, password: mirror.Password, excludeFile: mirror.ExcludeFile, - workingDir: filepath.Join(mirrorDir, mirror.Name), + workingDir: mirrorDir, logDir: logDir, logFile: filepath.Join(logDir, "latest.log"), useIPv6: mirror.UseIPv6, @@ -120,9 +133,10 @@ func (w *Worker) initProviders() { name: mirror.Name, stage1Profile: mirror.Stage1Profile, upstreamURL: mirror.Upstream, + rsyncCmd: mirror.Command, password: mirror.Password, excludeFile: mirror.ExcludeFile, - workingDir: filepath.Join(mirrorDir, mirror.Name), + workingDir: mirrorDir, logDir: logDir, logFile: filepath.Join(logDir, "latest.log"), useIPv6: mirror.UseIPv6, @@ -303,7 +317,7 @@ func (w *Worker) registorWorker() { func (w *Worker) updateStatus(jobMsg jobMessage) { url := fmt.Sprintf( - "%s/%s/jobs/%s", + "%s/workers/%s/jobs/%s", w.cfg.Manager.APIBase, w.Name(), jobMsg.name, @@ -329,7 +343,7 @@ func (w *Worker) fetchJobStatus() []MirrorStatus { var mirrorList []MirrorStatus url := fmt.Sprintf( - "%s/%s/jobs", + "%s/workers/%s/jobs", w.cfg.Manager.APIBase, w.Name(), ) From 72d9f87711763b87378b57bcaca9db00e8e143c9 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 21:04:23 +0800 Subject: [PATCH 47/66] tests: test files --- tests/bin/myrsync.sh | 3 +++ tests/bin/myrsync2.sh | 8 ++++++++ tests/managerMain.go | 23 +++++++++++++++++++++ tests/req.cnf | 3 ++- tests/rootCA.srl | 2 +- tests/worker.conf | 48 +++++++++++++++++++++++++++++++++++++++++++ tests/worker.crt | 22 ++++++++++++++++++++ tests/worker.csr | 18 ++++++++++++++++ tests/worker.key | 27 ++++++++++++++++++++++++ tests/workerMain.go | 17 +++++++++++++++ 10 files changed, 169 insertions(+), 2 deletions(-) create mode 100755 tests/bin/myrsync.sh create mode 100755 tests/bin/myrsync2.sh create mode 100644 tests/managerMain.go create mode 100644 tests/worker.conf create mode 100644 tests/worker.crt create mode 100644 tests/worker.csr create mode 100644 tests/worker.key create mode 100644 tests/workerMain.go diff --git a/tests/bin/myrsync.sh b/tests/bin/myrsync.sh new file mode 100755 index 0000000..e4039f7 --- /dev/null +++ b/tests/bin/myrsync.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo $@ +sleep 5 diff --git a/tests/bin/myrsync2.sh b/tests/bin/myrsync2.sh new file mode 100755 index 0000000..7be6e6a --- /dev/null +++ b/tests/bin/myrsync2.sh @@ -0,0 +1,8 @@ +#!/bin/bash +echo $TUNASYNC_WORKING_DIR +echo $TUNASYNC_LOG_FILE +echo $TUNASYNC_UPSTREAM_URL +echo $TUNASYNC_WORKING_DIR +echo $@ +sleep 5 +exit 1 diff --git a/tests/managerMain.go b/tests/managerMain.go new file mode 100644 index 0000000..f468287 --- /dev/null +++ b/tests/managerMain.go @@ -0,0 +1,23 @@ +package main + +import "github.com/tuna/tunasync/manager" + +var cfg = manager.Config{ + Debug: true, + Server: manager.ServerConfig{ + Addr: "127.0.0.1", + Port: 12345, + SSLCert: "manager.crt", + SSLKey: "manager.key", + }, + Files: manager.FileConfig{ + DBType: "bolt", + DBFile: "/tmp/tunasync/manager.db", + CACert: "rootCA.crt", + }, +} + +func main() { + m := manager.GetTUNASyncManager(&cfg) + m.Run() +} diff --git a/tests/req.cnf b/tests/req.cnf index 246fe72..ce4490d 100644 --- a/tests/req.cnf +++ b/tests/req.cnf @@ -23,4 +23,5 @@ subjectAltName = @alt_names [alt_names] DNS.1 = localhost -DNS.2 = manager.localhost +# DNS.2 = manager.localhost +DNS.2 = worker.localhost diff --git a/tests/rootCA.srl b/tests/rootCA.srl index 7e92478..e7d8f61 100644 --- a/tests/rootCA.srl +++ b/tests/rootCA.srl @@ -1 +1 @@ -DB01B233C4550DC2 +DB01B233C4550DC3 diff --git a/tests/worker.conf b/tests/worker.conf new file mode 100644 index 0000000..0f8f42a --- /dev/null +++ b/tests/worker.conf @@ -0,0 +1,48 @@ +[global] +name = "test_worker" +log_dir = "/tmp/tunasync/log/tunasync/{{.Name}}" +mirror_dir = "/tmp/tunasync" +concurrent = 10 +interval = 1 + +[manager] +api_base = "https://localhost:12345" +token = "some_token" +ca_cert = "rootCA.crt" + +[server] +hostname = "localhost" +listen_addr = "127.0.0.1" +listen_port = 6000 +ssl_cert = "worker.crt" +ssl_key = "worker.key" + +[[mirrors]] +name = "AOSP" +provider = "command" +command = "/tmp/tunasync/bin/myrsync2.sh" +upstream = "https://aosp.google.com/" +interval = 2 +mirror_dir = "/tmp/tunasync/git/AOSP" + [mirrors.env] + REPO = "/usr/local/bin/aosp-repo" + +[[mirrors]] +name = "debian" +command = "/tmp/tunasync/bin/myrsync.sh" +provider = "two-stage-rsync" +stage1_profile = "debian" +upstream = "rsync://ftp.debian.org/debian/" +use_ipv6 = true + + +[[mirrors]] +name = "fedora" +command = "/tmp/tunasync/bin/myrsync.sh" +provider = "rsync" +upstream = "rsync://ftp.fedoraproject.org/fedora/" +use_ipv6 = true +exclude_file = "/etc/tunasync.d/fedora-exclude.txt" + + +# vim: ft=toml diff --git a/tests/worker.crt b/tests/worker.crt new file mode 100644 index 0000000..9d9eab5 --- /dev/null +++ b/tests/worker.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDmTCCAoGgAwIBAgIJANsBsjPEVQ3DMA0GCSqGSIb3DQEBCwUAMIGEMQswCQYD +VQQGEwJDTjELMAkGA1UECAwCQkoxETAPBgNVBAcMCFRzaW5naHVhMQ0wCwYDVQQK +DARUVU5BMRAwDgYDVQQLDAdNaXJyb3JzMRIwEAYDVQQDDAlsb2NhbGhvc3QxIDAe +BgkqhkiG9w0BCQEWEXJvb3RAbWlycm9ycy50dW5hMB4XDTE2MDQyODEyMjEwMFoX +DTE3MDQyODEyMjEwMFowTzELMAkGA1UEBhMCQ04xCzAJBgNVBAgMAkJKMRAwDgYD +VQQHDAdCZWlqaW5nMQ0wCwYDVQQLDARUVU5BMRIwEAYDVQQDDAlsb2NhbGhvc3Qw +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCexn2BKhy7DGoFNNi05DOr +AZg/JITCxWJzrGMT0Ca5twP7yYTsrLDlbYhy2FwVQ45D1OycKKiOuzyxqV7lvgDI +iNtf3LYeEKImsuMxcjkDncQ1eY5kcNG/e0sAj9FyoK/pPbjbEzzfj5z5FqDxtYkf +4y5DR1pUf5SfQEJ0n5AclcXY8PrUwzA6MD6sAs4SZopQPunx3m0b1uYPACBIKiY0 +wZiUhrjoPCqR0orj8ZLDO0pGDFh8jmFFQMHNpwad37K3MXWkpAsR+MUXckocQ8O/ +6vIgFFDoqYxOuS3GkQ/Dh7dNaPhJ86OFJ+A8C0BDqHNYvkVVvA2gPmHN+8LFJHat +AgMBAAGjQjBAMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgXgMCYGA1UdEQQfMB2CCWxv +Y2FsaG9zdIIQd29ya2VyLmxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAECje +0jI8cp5eQyDzuHbbVcl8jZXMn/UVuHOQ/VLcaBOUWHDl7QANTTtyyYT+2Q+CdpyJ +Gn+fUB4tQP7naGR4bNpVytdttOlNZ89scZ3O74GX0vcAPvr62MxeASw44WuT6ir3 +zSTrww3qvvExG22atRIyGIFKLgmMMyzMskUFjELq80/nY55bCbStvhMJ0GHsC22n +2YRYD8+gyCJUT3hYjXymaPojvE9Cq6zBOUUP2yIwId2LQev2UNvJaEVvphmYtS08 +VVLiXy9ye6pc+0cZonJ4aTESRIgv53pPoHNhhRkR1xbdojUKhk0Fq8NKi2bPZVzQ +zVC9pCxHNGqRIcctzA== +-----END CERTIFICATE----- diff --git a/tests/worker.csr b/tests/worker.csr new file mode 100644 index 0000000..e95cb01 --- /dev/null +++ b/tests/worker.csr @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC5TCCAc0CAQAwTzELMAkGA1UEBhMCQ04xCzAJBgNVBAgMAkJKMRAwDgYDVQQH +DAdCZWlqaW5nMQ0wCwYDVQQLDARUVU5BMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCexn2BKhy7DGoFNNi05DOrAZg/ +JITCxWJzrGMT0Ca5twP7yYTsrLDlbYhy2FwVQ45D1OycKKiOuzyxqV7lvgDIiNtf +3LYeEKImsuMxcjkDncQ1eY5kcNG/e0sAj9FyoK/pPbjbEzzfj5z5FqDxtYkf4y5D +R1pUf5SfQEJ0n5AclcXY8PrUwzA6MD6sAs4SZopQPunx3m0b1uYPACBIKiY0wZiU +hrjoPCqR0orj8ZLDO0pGDFh8jmFFQMHNpwad37K3MXWkpAsR+MUXckocQ8O/6vIg +FFDoqYxOuS3GkQ/Dh7dNaPhJ86OFJ+A8C0BDqHNYvkVVvA2gPmHN+8LFJHatAgMB +AAGgUTBPBgkqhkiG9w0BCQ4xQjBAMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgXgMCYG +A1UdEQQfMB2CCWxvY2FsaG9zdIIQd29ya2VyLmxvY2FsaG9zdDANBgkqhkiG9w0B +AQsFAAOCAQEAjiJVwuZFuuNvVTGwiLxJgqGKCp2NMPFtlqD4snpTVzSgzJLSqBvl +d4CoF+ayW+4tY3HTmjUmWKuVZ/PC+MMWXd5LxfZC06u8uLXp2liUmD1NGqK1u6VD +gVcS2NyX/BhIYWp3ey61i25dHDcaY1MHto6zJ2kfnt0RunvaKr3jVKsZTrfqypfz +1AQ/E4SwdWRKaG1RorYgIs+G51oizCLoPIxMcipM+ub0Z00jfS7jFyPqtxcrtM+v +fpRIGlqW0jBWxJUQKpds7TkPrxVojZINaANsVk3Zw+TYvmurRyU8WPoilIyQ7vxF +tUSyxm2ss2B0tEqQZQytnNQut9G4s6svZg== +-----END CERTIFICATE REQUEST----- diff --git a/tests/worker.key b/tests/worker.key new file mode 100644 index 0000000..714ca51 --- /dev/null +++ b/tests/worker.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAnsZ9gSocuwxqBTTYtOQzqwGYPySEwsVic6xjE9AmubcD+8mE +7Kyw5W2IcthcFUOOQ9TsnCiojrs8sale5b4AyIjbX9y2HhCiJrLjMXI5A53ENXmO +ZHDRv3tLAI/RcqCv6T242xM834+c+Rag8bWJH+MuQ0daVH+Un0BCdJ+QHJXF2PD6 +1MMwOjA+rALOEmaKUD7p8d5tG9bmDwAgSComNMGYlIa46DwqkdKK4/GSwztKRgxY +fI5hRUDBzacGnd+ytzF1pKQLEfjFF3JKHEPDv+ryIBRQ6KmMTrktxpEPw4e3TWj4 +SfOjhSfgPAtAQ6hzWL5FVbwNoD5hzfvCxSR2rQIDAQABAoIBAG37hrJzTmWPSt2C +Zt6e+N9rAmAy1rfobLM95X+y/zBEA0FlrWsYkIzMW+lZ0Cd2nVSFaMUfMOt17awP +a8nu3LIMgxGbXJfk4720ysXUnoPPxDtakXUn5VMjf6fK98XUYyZI+AThBZjC7XRp +5WCpZGwvPTujcIH5wiSyKZaJdRUm3wpoZ1NB3VcmxoQM72yleU2t79YsNyFavbcn +z6/1zaz4q1BVGZpioD9WBPGAhktrwmgYL3xcrqvMeGSY281bbXgV/YySIxibBa9z +bTq4dImT4CxNzx2y2A+b9n/zR7TBitww1yvCf7OPJ0NK5keEVtef0p2TscjOlndk +mv9/NQECgYEAy+2rdapdTgafYu1tM9lhx1VJjQZ8jpjkYKVzoknQ/m/4c2adYsnz +LsowkCo/0DpjxVPE/yo6wEBUct0A7/dbQCSXhx/XStjuIUT4mZjOXtBtLKrJSF8y +WzhFyiPv3+wdbxCmrbfK8/z+UWa+rcIV7saCbDJJTTkT6E32dBNW0O0CgYEAx1FF +Eg+5SeqYQM9i8A708ySxPrFsRY1i2MVIiSkLiN7MEJAJKgAl8xn0/0pGDD/qjWlc +2nL7YzYoWOGnJAfqUF5OlWZ3+VOBYEHJIrA2ajgdjVYhnfz7zCZy51OanoVJDBjw +2gQWnBC0ISeygf4NhyvLianwoc1cp+BgVQm6RMECgYEAnF3ldxfm64lQdb6wWW15 ++CqBd01d/MlndGPpQqtvQWoCDBrG25UWju4iRqjevX/IOOp+x1lOK1QobNrheR8m +LQzh046quo2UKpaEOOJee309+V4LcR7tsdx4RwM/T2fxOdR+uf2P9X4sU6aA1yNX +RfuYzfXRFxGJHjuJmn+pthECgYEAvf1jv3GphyHNe4mzn2xCZTpGkaIBuNKqtEJp +gATV7+Of1PHXKmf1xKKrfGVKHAcZBy61yazsn4dSMlb2QUwiN/WNJrAEEG9e1Wgf +16bsV5eh48WESdqKEfFcedChhBU8qgFkJAzdmGn7qdbzOyH1tzEx1MlejHz6ozMn +4CdjnIECgYBAEquvEj6eptAx+tVk4bk/XE0XT2qC6kYCB3U08hhlSTCb2EoDPm+n +/gEpvHH3+pz4jvUDoBMvL4uncoUQQuVP4rvv3PoElAtl1bT1mKovqqUFJTXqZEK9 +bBgGkvCi5HpeCocIFgLxyjajnhBEeMEBkcfkG7SNrOtMTUc/dUWKaA== +-----END RSA PRIVATE KEY----- diff --git a/tests/workerMain.go b/tests/workerMain.go new file mode 100644 index 0000000..24bf6a2 --- /dev/null +++ b/tests/workerMain.go @@ -0,0 +1,17 @@ +package main + +import ( + "fmt" + + "github.com/tuna/tunasync/worker" +) + +func main() { + cfg, err := worker.LoadConfig("worker.conf") + if err != nil { + fmt.Println(err.Error()) + return + } + m := worker.GetTUNASyncWorker(cfg) + m.Run() +} From 3874d41afc706f771936e04a9f34ac33a9e6cc5f Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 21:09:21 +0800 Subject: [PATCH 48/66] refactor(manager): let manager export LoadConfig --- manager/config.go | 6 +++++- manager/config_test.go | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/manager/config.go b/manager/config.go index 155ee95..ca1f00f 100644 --- a/manager/config.go +++ b/manager/config.go @@ -29,7 +29,7 @@ type FileConfig struct { CACert string `toml:"ca_cert"` } -func loadConfig(cfgFile string, c *cli.Context) (*Config, error) { +func LoadConfig(cfgFile string, c *cli.Context) (*Config, error) { cfg := new(Config) cfg.Server.Addr = "127.0.0.1" @@ -46,6 +46,10 @@ func loadConfig(cfgFile string, c *cli.Context) (*Config, error) { } } + if c == nil { + return cfg, nil + } + if c.String("addr") != "" { cfg.Server.Addr = c.String("addr") } diff --git a/manager/config_test.go b/manager/config_test.go index 4a81347..a7873c2 100644 --- a/manager/config_test.go +++ b/manager/config_test.go @@ -72,7 +72,7 @@ func TestConfig(t *testing.T) { Convey("when giving no config options", func() { app.Action = func(c *cli.Context) { cfgFile := c.String("config") - cfg, err := loadConfig(cfgFile, c) + cfg, err := LoadConfig(cfgFile, c) So(err, ShouldEqual, nil) So(cfg.Server.Addr, ShouldEqual, "127.0.0.1") } @@ -83,7 +83,7 @@ func TestConfig(t *testing.T) { app.Action = func(c *cli.Context) { cfgFile := c.String("config") So(cfgFile, ShouldEqual, tmpfile.Name()) - conf, err := loadConfig(cfgFile, c) + conf, err := LoadConfig(cfgFile, c) So(err, ShouldEqual, nil) So(conf.Server.Addr, ShouldEqual, "0.0.0.0") So(conf.Server.Port, ShouldEqual, 5000) @@ -99,7 +99,7 @@ func TestConfig(t *testing.T) { app.Action = func(c *cli.Context) { cfgFile := c.String("config") So(cfgFile, ShouldEqual, "") - conf, err := loadConfig(cfgFile, c) + conf, err := LoadConfig(cfgFile, c) So(err, ShouldEqual, nil) So(conf.Server.Addr, ShouldEqual, "0.0.0.0") So(conf.Server.Port, ShouldEqual, 5001) @@ -119,7 +119,7 @@ func TestConfig(t *testing.T) { app.Action = func(c *cli.Context) { cfgFile := c.String("config") So(cfgFile, ShouldEqual, tmpfile.Name()) - conf, err := loadConfig(cfgFile, c) + conf, err := LoadConfig(cfgFile, c) So(err, ShouldEqual, nil) So(conf.Server.Addr, ShouldEqual, "0.0.0.0") So(conf.Server.Port, ShouldEqual, 5000) From 5f7872293622cd667e7030a20f75ec49bc3a416d Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 21:09:33 +0800 Subject: [PATCH 49/66] tests: update manager tests --- tests/manager.conf | 15 +++++++++++++++ tests/managerMain.go | 26 ++++++++++---------------- 2 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 tests/manager.conf diff --git a/tests/manager.conf b/tests/manager.conf new file mode 100644 index 0000000..3f6a45f --- /dev/null +++ b/tests/manager.conf @@ -0,0 +1,15 @@ +debug = true + +[server] +addr = "127.0.0.1" +port = 12345 +ssl_cert = "manager.crt" +ssl_key = "manager.key" + +[files] +db_type = "bolt" +db_file = "/tmp/tunasync/manager.db" +ca_cert = "rootCA.crt" + + +# vim: ft=toml diff --git a/tests/managerMain.go b/tests/managerMain.go index f468287..4443019 100644 --- a/tests/managerMain.go +++ b/tests/managerMain.go @@ -1,23 +1,17 @@ package main -import "github.com/tuna/tunasync/manager" +import ( + "fmt" -var cfg = manager.Config{ - Debug: true, - Server: manager.ServerConfig{ - Addr: "127.0.0.1", - Port: 12345, - SSLCert: "manager.crt", - SSLKey: "manager.key", - }, - Files: manager.FileConfig{ - DBType: "bolt", - DBFile: "/tmp/tunasync/manager.db", - CACert: "rootCA.crt", - }, -} + "github.com/tuna/tunasync/manager" +) func main() { - m := manager.GetTUNASyncManager(&cfg) + cfg, err := manager.LoadConfig("manager.conf", nil) + if err != nil { + fmt.Println(err.Error()) + return + } + m := manager.GetTUNASyncManager(cfg) m.Run() } From 292a24ba20b19615727506c0a5708a5ac41f0f82 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 22:46:53 +0800 Subject: [PATCH 50/66] fix(worker): fixed job status and control logic --- internal/msg.go | 28 ++++++++++++- manager/server.go | 34 +++++++++++++++ worker/job.go | 80 ++++++++++++++++++++---------------- worker/provider.go | 1 + worker/schedule.go | 1 + worker/worker.go | 100 ++++++++++++++++++++++++++++++--------------- 6 files changed, 174 insertions(+), 70 deletions(-) diff --git a/internal/msg.go b/internal/msg.go index a433949..85337d6 100644 --- a/internal/msg.go +++ b/internal/msg.go @@ -1,6 +1,9 @@ package internal -import "time" +import ( + "fmt" + "time" +) // A StatusUpdateMsg represents a msg when // a worker has done syncing @@ -34,6 +37,22 @@ const ( CmdPing // ensure the goroutine is alive ) +func (c CmdVerb) String() string { + switch c { + case CmdStart: + return "start" + case CmdStop: + return "stop" + case CmdDisable: + return "disable" + case CmdRestart: + return "restart" + case CmdPing: + return "ping" + } + return "unknown" +} + // A WorkerCmd is the command message send from the // manager to a worker type WorkerCmd struct { @@ -42,6 +61,13 @@ type WorkerCmd struct { Args []string `json:"args"` } +func (c WorkerCmd) String() string { + if len(c.Args) > 0 { + return fmt.Sprintf("%v (%s, %v)", c.Cmd, c.MirrorID, c.Args) + } + return fmt.Sprintf("%v (%s)", c.Cmd, c.MirrorID) +} + // A ClientCmd is the command message send from client // to the manager type ClientCmd struct { diff --git a/manager/server.go b/manager/server.go index 94f40ad..147cecb 100644 --- a/manager/server.go +++ b/manager/server.go @@ -194,6 +194,24 @@ func (s *Manager) updateJobOfWorker(c *gin.Context) { var status MirrorStatus c.BindJSON(&status) mirrorName := status.Name + + curStatus, err := s.adapter.GetMirrorStatus(workerID, mirrorName) + if err != nil { + err := fmt.Errorf("failed to get job %s of worker %s: %s", + mirrorName, workerID, err.Error(), + ) + c.Error(err) + s.returnErrJSON(c, http.StatusInternalServerError, err) + return + } + + // Only successful syncing needs last_update + if status.Status == Success { + status.LastUpdate = time.Now() + } else { + status.LastUpdate = curStatus.LastUpdate + } + newStatus, err := s.adapter.UpdateMirrorStatus(workerID, mirrorName, status) if err != nil { err := fmt.Errorf("failed to update job %s of worker %s: %s", @@ -231,6 +249,22 @@ func (s *Manager) handleClientCmd(c *gin.Context) { Args: clientCmd.Args, } + // update job status, even if the job did not disable successfully, + // this status should be set as disabled + curStat, _ := s.adapter.GetMirrorStatus(clientCmd.WorkerID, clientCmd.MirrorID) + changed := false + switch clientCmd.Cmd { + case CmdDisable: + curStat.Status = Disabled + changed = true + case CmdStop: + curStat.Status = Paused + changed = true + } + if changed { + s.adapter.UpdateMirrorStatus(clientCmd.WorkerID, clientCmd.MirrorID, curStat) + } + // post command to worker _, err = PostJSON(workerURL, workerCmd, s.tlsConfig) if err != nil { diff --git a/worker/job.go b/worker/job.go index f71136a..de5f79a 100644 --- a/worker/job.go +++ b/worker/job.go @@ -20,23 +20,28 @@ const ( ) type jobMessage struct { - status tunasync.SyncStatus - name string - msg string + status tunasync.SyncStatus + name string + msg string + schedule bool } type mirrorJob struct { - provider mirrorProvider - ctrlChan chan ctrlAction - disabled chan empty - enabled bool + provider mirrorProvider + ctrlChan chan ctrlAction + disabled chan empty + started bool + schedule bool + isDisabled bool } func newMirrorJob(provider mirrorProvider) *mirrorJob { return &mirrorJob{ - provider: provider, - ctrlChan: make(chan ctrlAction, 1), - enabled: false, + provider: provider, + ctrlChan: make(chan ctrlAction, 1), + started: false, + schedule: false, + isDisabled: false, } } @@ -44,18 +49,6 @@ func (m *mirrorJob) Name() string { return m.provider.Name() } -func (m *mirrorJob) Disabled() bool { - if !m.enabled { - return true - } - select { - case <-m.disabled: - return true - default: - return false - } -} - // runMirrorJob is the goroutine where syncing job runs in // arguments: // provider: mirror provider object @@ -66,7 +59,11 @@ func (m *mirrorJob) Disabled() bool { func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) error { m.disabled = make(chan empty) - defer close(m.disabled) + defer func() { + close(m.disabled) + m.schedule = false + m.isDisabled = true + }() provider := m.provider @@ -81,6 +78,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err managerChan <- jobMessage{ tunasync.Failed, m.Name(), fmt.Sprintf("error exec hook %s: %s", hookname, err.Error()), + false, } return err } @@ -91,7 +89,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err runJobWrapper := func(kill <-chan empty, jobDone chan<- empty) error { defer close(jobDone) - managerChan <- jobMessage{tunasync.PreSyncing, m.Name(), ""} + managerChan <- jobMessage{tunasync.PreSyncing, m.Name(), "", false} logger.Info("start syncing: %s", m.Name()) Hooks := provider.Hooks() @@ -118,7 +116,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err } // start syncing - managerChan <- jobMessage{tunasync.Syncing, m.Name(), ""} + managerChan <- jobMessage{tunasync.Syncing, m.Name(), "", false} var syncErr error syncDone := make(chan error, 1) @@ -152,7 +150,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err if syncErr == nil { // syncing success logger.Info("succeeded syncing %s", m.Name()) - managerChan <- jobMessage{tunasync.Success, m.Name(), ""} + managerChan <- jobMessage{tunasync.Success, m.Name(), "", true} // post-success hooks err := runHooks(rHooks, func(h jobHook) error { return h.postSuccess() }, "post-success") if err != nil { @@ -164,7 +162,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err // syncing failed logger.Warning("failed syncing %s: %s", m.Name(), syncErr.Error()) - managerChan <- jobMessage{tunasync.Failed, m.Name(), syncErr.Error()} + managerChan <- jobMessage{tunasync.Failed, m.Name(), syncErr.Error(), retry == maxRetry-1} // post-fail hooks logger.Debug("post-fail hooks") @@ -194,7 +192,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err } for { - if m.enabled { + if m.started { kill := make(chan empty) jobDone := make(chan empty) go runJob(kill, jobDone) @@ -206,21 +204,24 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err case ctrl := <-m.ctrlChan: switch ctrl { case jobStop: - m.enabled = false + m.schedule = false + m.started = false close(kill) <-jobDone case jobDisable: - m.enabled = false + m.schedule = false + m.isDisabled = true + m.started = false close(kill) <-jobDone return nil case jobRestart: - m.enabled = true + m.started = true close(kill) <-jobDone continue case jobStart: - m.enabled = true + m.started = true goto _wait_for_job default: // TODO: implement this @@ -233,14 +234,21 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err ctrl := <-m.ctrlChan switch ctrl { case jobStop: - m.enabled = false + m.schedule = false + m.started = false case jobDisable: - m.enabled = false + m.schedule = false + m.isDisabled = true + m.started = false return nil case jobRestart: - m.enabled = true + m.schedule = true + m.isDisabled = false + m.started = true case jobStart: - m.enabled = true + m.schedule = true + m.isDisabled = false + m.started = true default: // TODO return nil diff --git a/worker/provider.go b/worker/provider.go index 498a3e7..cf757f0 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -85,6 +85,7 @@ func (p *baseProvider) Context() *Context { } func (p *baseProvider) Interval() time.Duration { + // logger.Debug("interval for %s: %v", p.Name(), p.interval) return p.interval } diff --git a/worker/schedule.go b/worker/schedule.go index cb95a5d..0d3f8f0 100644 --- a/worker/schedule.go +++ b/worker/schedule.go @@ -44,6 +44,7 @@ func (q *scheduleQueue) Pop() *mirrorJob { defer first.Close() t := first.Key().(time.Time) + // logger.Debug("First job should run @%v", t) if t.Before(time.Now()) { job := first.Value().(*mirrorJob) q.list.Delete(first.Key()) diff --git a/worker/worker.go b/worker/worker.go index 56bffa6..40eeaa6 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -28,8 +28,6 @@ type Worker struct { schedule *scheduleQueue httpServer *gin.Engine tlsConfig *tls.Config - - mirrorStatus map[string]SyncStatus } // GetTUNASyncWorker returns a singalton worker @@ -46,8 +44,7 @@ func GetTUNASyncWorker(cfg *Config) *Worker { managerChan: make(chan jobMessage, 32), semaphore: make(chan empty, cfg.Global.Concurrent), - schedule: newScheduleQueue(), - mirrorStatus: make(map[string]SyncStatus), + schedule: newScheduleQueue(), } if cfg.Manager.CACert != "" { @@ -89,6 +86,9 @@ func (w *Worker) initProviders() { c.Global.MirrorDir, mirror.Name, ) } + if mirror.Interval == 0 { + mirror.Interval = c.Global.Interval + } logDir = formatLogDir(logDir, mirror) var provider mirrorProvider @@ -163,8 +163,6 @@ func (w *Worker) initJobs() { for name, provider := range w.providers { w.jobs[name] = newMirrorJob(provider) - go w.jobs[name].Run(w.managerChan, w.semaphore) - w.mirrorStatus[name] = Paused } } @@ -185,24 +183,40 @@ func (w *Worker) makeHTTPServer() { c.JSON(http.StatusNotFound, gin.H{"msg": fmt.Sprintf("Mirror ``%s'' not found", cmd.MirrorID)}) return } + logger.Info("Received command: %v", cmd) // if job disabled, start them first switch cmd.Cmd { case CmdStart, CmdRestart: - if job.Disabled() { + if job.isDisabled { go job.Run(w.managerChan, w.semaphore) } } switch cmd.Cmd { case CmdStart: + job.schedule = true + job.isDisabled = false job.ctrlChan <- jobStart - case CmdStop: - job.ctrlChan <- jobStop case CmdRestart: + job.schedule = true + job.isDisabled = false job.ctrlChan <- jobRestart + case CmdStop: + // if job is disabled, no goroutine would be there + // receiving this signal + if !job.isDisabled { + job.schedule = false + job.isDisabled = false + w.schedule.Remove(job.Name()) + job.ctrlChan <- jobStop + } case CmdDisable: - w.schedule.Remove(job.Name()) - job.ctrlChan <- jobDisable - <-job.disabled + if !job.isDisabled { + job.schedule = false + job.isDisabled = true + w.schedule.Remove(job.Name()) + job.ctrlChan <- jobDisable + <-job.disabled + } case CmdPing: job.ctrlChan <- jobStart default: @@ -243,15 +257,32 @@ func (w *Worker) runSchedule() { for name := range w.jobs { unset[name] = true } + // Fetch mirror list stored in the manager + // put it on the scheduled time + // if it's disabled, ignore it for _, m := range mirrorList { if job, ok := w.jobs[m.Name]; ok { - stime := m.LastUpdate.Add(job.provider.Interval()) - w.schedule.AddJob(stime, job) delete(unset, m.Name) + switch m.Status { + case Paused: + go job.Run(w.managerChan, w.semaphore) + job.schedule = false + continue + case Disabled: + job.schedule = false + job.isDisabled = true + continue + default: + go job.Run(w.managerChan, w.semaphore) + stime := m.LastUpdate.Add(job.provider.Interval()) + logger.Debug("Scheduling job %s @%s", job.Name(), stime.Format("2006-01-02 15:04:05")) + w.schedule.AddJob(stime, job) + } } } for name := range unset { job := w.jobs[name] + go job.Run(w.managerChan, w.semaphore) w.schedule.AddJob(time.Now(), job) } @@ -259,22 +290,26 @@ func (w *Worker) runSchedule() { select { case jobMsg := <-w.managerChan: // got status update from job - w.updateStatus(jobMsg) - status := w.mirrorStatus[jobMsg.name] - if status == Disabled || status == Paused { + job := w.jobs[jobMsg.name] + if !job.schedule { + logger.Info("Job %s disabled/paused, skip adding new schedule", jobMsg.name) continue } - w.mirrorStatus[jobMsg.name] = jobMsg.status - switch jobMsg.status { - case Success, Failed: - job := w.jobs[jobMsg.name] - w.schedule.AddJob( - time.Now().Add(job.provider.Interval()), - job, + + w.updateStatus(jobMsg) + + if jobMsg.schedule { + schedTime := time.Now().Add(job.provider.Interval()) + logger.Info( + "Next scheduled time for %s: %s", + job.Name(), + schedTime.Format("2006-01-02 15:04:05"), ) + w.schedule.AddJob(schedTime, job) } - case <-time.Tick(10 * time.Second): + case <-time.Tick(5 * time.Second): + // check schedule every 5 seconds if job := w.schedule.Pop(); job != nil { job.ctrlChan <- jobStart } @@ -324,14 +359,13 @@ func (w *Worker) updateStatus(jobMsg jobMessage) { ) p := w.providers[jobMsg.name] smsg := MirrorStatus{ - Name: jobMsg.name, - Worker: w.cfg.Global.Name, - IsMaster: true, - Status: jobMsg.status, - LastUpdate: time.Now(), - Upstream: p.Upstream(), - Size: "unknown", - ErrorMsg: jobMsg.msg, + Name: jobMsg.name, + Worker: w.cfg.Global.Name, + IsMaster: true, + Status: jobMsg.status, + Upstream: p.Upstream(), + Size: "unknown", + ErrorMsg: jobMsg.msg, } if _, err := PostJSON(url, smsg, w.tlsConfig); err != nil { From ad2b65fcaa264d0174434ed4fdbb857c2b08010e Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 23:12:10 +0800 Subject: [PATCH 51/66] fix: server test --- manager/server.go | 10 +-------- manager/server_test.go | 50 +++++++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/manager/server.go b/manager/server.go index 147cecb..30f77bf 100644 --- a/manager/server.go +++ b/manager/server.go @@ -195,15 +195,7 @@ func (s *Manager) updateJobOfWorker(c *gin.Context) { c.BindJSON(&status) mirrorName := status.Name - curStatus, err := s.adapter.GetMirrorStatus(workerID, mirrorName) - if err != nil { - err := fmt.Errorf("failed to get job %s of worker %s: %s", - mirrorName, workerID, err.Error(), - ) - c.Error(err) - s.returnErrJSON(c, http.StatusInternalServerError, err) - return - } + curStatus, _ := s.adapter.GetMirrorStatus(workerID, mirrorName) // Only successful syncing needs last_update if status.Status == Success { diff --git a/manager/server_test.go b/manager/server_test.go index 1cc9259..61e67a2 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -82,13 +82,12 @@ func TestHTTPServer(t *testing.T) { Convey("update mirror status of a existed worker", func(ctx C) { status := MirrorStatus{ - Name: "arch-sync1", - Worker: "test_worker1", - IsMaster: true, - Status: Success, - LastUpdate: time.Now(), - Upstream: "mirrors.tuna.tsinghua.edu.cn", - Size: "3GB", + Name: "arch-sync1", + Worker: "test_worker1", + IsMaster: true, + Status: Success, + Upstream: "mirrors.tuna.tsinghua.edu.cn", + Size: "3GB", } resp, err := PostJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", baseURL, status.Worker, status.Name), status, nil) defer resp.Body.Close() @@ -96,31 +95,36 @@ func TestHTTPServer(t *testing.T) { So(resp.StatusCode, ShouldEqual, http.StatusOK) Convey("list mirror status of an existed worker", func(ctx C) { + var ms []MirrorStatus + resp, err := GetJSON(baseURL+"/workers/test_worker1/jobs", &ms, nil) - expectedResponse, err := json.Marshal([]MirrorStatus{status}) - So(err, ShouldBeNil) - resp, err := http.Get(baseURL + "/workers/test_worker1/jobs") So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) // err = json.NewDecoder(resp.Body).Decode(&mirrorStatusList) - body, err := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - So(err, ShouldBeNil) - So(strings.TrimSpace(string(body)), ShouldEqual, string(expectedResponse)) + m := ms[0] + So(m.Name, ShouldEqual, status.Name) + So(m.Worker, ShouldEqual, status.Worker) + So(m.Status, ShouldEqual, status.Status) + So(m.Upstream, ShouldEqual, status.Upstream) + So(m.Size, ShouldEqual, status.Size) + So(m.IsMaster, ShouldEqual, status.IsMaster) + So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second) + }) Convey("list all job status of all workers", func(ctx C) { - expectedResponse, err := json.Marshal( - []webMirrorStatus{convertMirrorStatus(status)}, - ) - So(err, ShouldBeNil) - resp, err := http.Get(baseURL + "/jobs") + var ms []webMirrorStatus + resp, err := GetJSON(baseURL+"/jobs", &ms, nil) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) - body, err := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - So(err, ShouldBeNil) - So(strings.TrimSpace(string(body)), ShouldEqual, string(expectedResponse)) + + m := ms[0] + So(m.Name, ShouldEqual, status.Name) + So(m.Status, ShouldEqual, status.Status) + So(m.Upstream, ShouldEqual, status.Upstream) + So(m.Size, ShouldEqual, status.Size) + So(m.IsMaster, ShouldEqual, status.IsMaster) + So(time.Now().Sub(m.LastUpdate.Time), ShouldBeLessThan, 1*time.Second) }) }) From 42c645a7369070735ed153c1bc5513e5abd80545 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 28 Apr 2016 23:12:21 +0800 Subject: [PATCH 52/66] chore: ignore building tests --- tests/httpClient.go | 2 ++ tests/httpServer.go | 2 ++ tests/managerMain.go | 2 ++ tests/workerMain.go | 2 ++ 4 files changed, 8 insertions(+) diff --git a/tests/httpClient.go b/tests/httpClient.go index a36fd85..b719282 100644 --- a/tests/httpClient.go +++ b/tests/httpClient.go @@ -1,3 +1,5 @@ +// +build ignore + package main import ( diff --git a/tests/httpServer.go b/tests/httpServer.go index 822eb56..45ab892 100644 --- a/tests/httpServer.go +++ b/tests/httpServer.go @@ -1,3 +1,5 @@ +// +build ignore + package main import ( diff --git a/tests/managerMain.go b/tests/managerMain.go index 4443019..0f6033b 100644 --- a/tests/managerMain.go +++ b/tests/managerMain.go @@ -1,3 +1,5 @@ +// +build ignore + package main import ( diff --git a/tests/workerMain.go b/tests/workerMain.go index 24bf6a2..61ed926 100644 --- a/tests/workerMain.go +++ b/tests/workerMain.go @@ -1,3 +1,5 @@ +// +build ignore + package main import ( From d1981379a4ac655e91fd31530ab7f7e5b3d500d0 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 07:59:54 +0800 Subject: [PATCH 53/66] fix(manager): timezone issue of status test --- manager/status.go | 4 ++-- manager/status_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/manager/status.go b/manager/status.go index 8df2b3c..31bd1d5 100644 --- a/manager/status.go +++ b/manager/status.go @@ -13,11 +13,11 @@ type textTime struct { } func (t textTime) MarshalJSON() ([]byte, error) { - return json.Marshal(t.Format("2006-01-02 15:04:05")) + return json.Marshal(t.Format("2006-01-02 15:04:05 -0700")) } func (t *textTime) UnmarshalJSON(b []byte) error { s := string(b) - t2, err := time.ParseInLocation(`"2006-01-02 15:04:05"`, s, time.Local) + t2, err := time.Parse(`"2006-01-02 15:04:05 -0700"`, s) *t = textTime{t2} return err } diff --git a/manager/status_test.go b/manager/status_test.go index 2ca0f1e..9cd046a 100644 --- a/manager/status_test.go +++ b/manager/status_test.go @@ -12,7 +12,7 @@ import ( func TestStatus(t *testing.T) { Convey("status json ser-de should work", t, func() { - tz := "Asia/Shanghai" + tz := "Asia/Tokyo" loc, err := time.LoadLocation(tz) So(err, ShouldBeNil) t := time.Date(2016, time.April, 16, 23, 8, 10, 0, loc) @@ -27,7 +27,7 @@ func TestStatus(t *testing.T) { b, err := json.Marshal(m) So(err, ShouldBeNil) - // fmt.Println(string(b)) + //fmt.Println(string(b)) var m2 webMirrorStatus err = json.Unmarshal(b, &m2) So(err, ShouldBeNil) From 2268eb3b0f4a61ede9278dae863a995fcdd3dec7 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 08:57:14 +0800 Subject: [PATCH 54/66] fix(tunasync): connection leakage caused by http keep-alive --- internal/util.go | 51 +++++++++++++++++++++++++++-------------------- manager/server.go | 36 +++++++++++++++++++-------------- worker/worker.go | 30 +++++++++++++++++----------- 3 files changed, 68 insertions(+), 49 deletions(-) diff --git a/internal/util.go b/internal/util.go index 9773914..80b21c8 100644 --- a/internal/util.go +++ b/internal/util.go @@ -8,6 +8,7 @@ import ( "errors" "io/ioutil" "net/http" + "time" ) // GetTLSConfig generate tls.Config from CAFile @@ -28,20 +29,34 @@ func GetTLSConfig(CAFile string) (*tls.Config, error) { return tlsConfig, nil } -// PostJSON posts json object to url -func PostJSON(url string, obj interface{}, tlsConfig *tls.Config) (*http.Response, error) { - var client *http.Client - if tlsConfig == nil { - client = &http.Client{} - } else { - tr := &http.Transport{ - TLSClientConfig: tlsConfig, - } - client = &http.Client{ - Transport: tr, +// CreateHTTPClient returns a http.Client +func CreateHTTPClient(CAFile string) (*http.Client, error) { + var tlsConfig *tls.Config + var err error + + if CAFile != "" { + tlsConfig, err = GetTLSConfig(CAFile) + if err != nil { + return nil, err } } + tr := &http.Transport{ + MaxIdleConnsPerHost: 20, + TLSClientConfig: tlsConfig, + } + + return &http.Client{ + Transport: tr, + Timeout: 5 * time.Second, + }, nil +} + +// PostJSON posts json object to url +func PostJSON(url string, obj interface{}, client *http.Client) (*http.Response, error) { + if client == nil { + client, _ = CreateHTTPClient("") + } b := new(bytes.Buffer) if err := json.NewEncoder(b).Encode(obj); err != nil { return nil, err @@ -50,17 +65,9 @@ func PostJSON(url string, obj interface{}, tlsConfig *tls.Config) (*http.Respons } // GetJSON gets a json response from url -func GetJSON(url string, obj interface{}, tlsConfig *tls.Config) (*http.Response, error) { - var client *http.Client - if tlsConfig == nil { - client = &http.Client{} - } else { - tr := &http.Transport{ - TLSClientConfig: tlsConfig, - } - client = &http.Client{ - Transport: tr, - } +func GetJSON(url string, obj interface{}, client *http.Client) (*http.Response, error) { + if client == nil { + client, _ = CreateHTTPClient("") } resp, err := client.Get(url) diff --git a/manager/server.go b/manager/server.go index 30f77bf..2d1ada9 100644 --- a/manager/server.go +++ b/manager/server.go @@ -1,7 +1,6 @@ package manager import ( - "crypto/tls" "fmt" "net/http" "time" @@ -20,10 +19,10 @@ var manager *Manager // A Manager represents a manager server type Manager struct { - cfg *Config - engine *gin.Engine - adapter dbAdapter - tlsConfig *tls.Config + cfg *Config + engine *gin.Engine + adapter dbAdapter + httpClient *http.Client } // GetTUNASyncManager returns the manager from config @@ -37,19 +36,18 @@ func GetTUNASyncManager(cfg *Config) *Manager { gin.SetMode(gin.ReleaseMode) } s := &Manager{ - cfg: cfg, - engine: gin.Default(), - adapter: nil, - tlsConfig: nil, + cfg: cfg, + engine: gin.Default(), + adapter: nil, } if cfg.Files.CACert != "" { - tlsConfig, err := GetTLSConfig(cfg.Files.CACert) + httpClient, err := CreateHTTPClient(cfg.Files.CACert) if err != nil { - logger.Error("Error initializing TLS config: %s", err.Error()) + logger.Error("Error initializing HTTP client: %s", err.Error()) return nil } - s.tlsConfig = tlsConfig + s.httpClient = httpClient } if cfg.Files.DBFile != "" { @@ -96,12 +94,20 @@ func (s *Manager) setDBAdapter(adapter dbAdapter) { // Run runs the manager server forever func (s *Manager) Run() { addr := fmt.Sprintf("%s:%d", s.cfg.Server.Addr, s.cfg.Server.Port) + + httpServer := &http.Server{ + Addr: addr, + Handler: s.engine, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + if s.cfg.Server.SSLCert == "" && s.cfg.Server.SSLKey == "" { - if err := s.engine.Run(addr); err != nil { + if err := httpServer.ListenAndServe(); err != nil { panic(err) } } else { - if err := s.engine.RunTLS(addr, s.cfg.Server.SSLCert, s.cfg.Server.SSLKey); err != nil { + if err := httpServer.ListenAndServeTLS(s.cfg.Server.SSLCert, s.cfg.Server.SSLKey); err != nil { panic(err) } } @@ -258,7 +264,7 @@ func (s *Manager) handleClientCmd(c *gin.Context) { } // post command to worker - _, err = PostJSON(workerURL, workerCmd, s.tlsConfig) + _, err = PostJSON(workerURL, workerCmd, s.httpClient) if err != nil { err := fmt.Errorf("post command to worker %s(%s) fail: %s", workerID, workerURL, err.Error()) c.Error(err) diff --git a/worker/worker.go b/worker/worker.go index 40eeaa6..9cd3f8a 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -2,7 +2,6 @@ package worker import ( "bytes" - "crypto/tls" "errors" "fmt" "html/template" @@ -26,8 +25,8 @@ type Worker struct { semaphore chan empty schedule *scheduleQueue - httpServer *gin.Engine - tlsConfig *tls.Config + httpEngine *gin.Engine + httpClient *http.Client } // GetTUNASyncWorker returns a singalton worker @@ -48,12 +47,12 @@ func GetTUNASyncWorker(cfg *Config) *Worker { } if cfg.Manager.CACert != "" { - tlsConfig, err := GetTLSConfig(cfg.Manager.CACert) + httpClient, err := CreateHTTPClient(cfg.Manager.CACert) if err != nil { - logger.Error("Failed to init TLS config: %s", err.Error()) + logger.Error("Error initializing HTTP client: %s", err.Error()) return nil } - w.tlsConfig = tlsConfig + w.httpClient = httpClient } w.initJobs() @@ -227,18 +226,25 @@ func (w *Worker) makeHTTPServer() { c.JSON(http.StatusOK, gin.H{"msg": "OK"}) }) - w.httpServer = s + w.httpEngine = s } func (w *Worker) runHTTPServer() { addr := fmt.Sprintf("%s:%d", w.cfg.Server.Addr, w.cfg.Server.Port) + httpServer := &http.Server{ + Addr: addr, + Handler: w.httpEngine, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + if w.cfg.Server.SSLCert == "" && w.cfg.Server.SSLKey == "" { - if err := w.httpServer.Run(addr); err != nil { + if err := httpServer.ListenAndServe(); err != nil { panic(err) } } else { - if err := w.httpServer.RunTLS(addr, w.cfg.Server.SSLCert, w.cfg.Server.SSLKey); err != nil { + if err := httpServer.ListenAndServeTLS(w.cfg.Server.SSLCert, w.cfg.Server.SSLKey); err != nil { panic(err) } } @@ -345,7 +351,7 @@ func (w *Worker) registorWorker() { URL: w.URL(), } - if _, err := PostJSON(url, msg, w.tlsConfig); err != nil { + if _, err := PostJSON(url, msg, w.httpClient); err != nil { logger.Error("Failed to register worker") } } @@ -368,7 +374,7 @@ func (w *Worker) updateStatus(jobMsg jobMessage) { ErrorMsg: jobMsg.msg, } - if _, err := PostJSON(url, smsg, w.tlsConfig); err != nil { + if _, err := PostJSON(url, smsg, w.httpClient); err != nil { logger.Error("Failed to update mirror(%s) status: %s", jobMsg.name, err.Error()) } } @@ -382,7 +388,7 @@ func (w *Worker) fetchJobStatus() []MirrorStatus { w.Name(), ) - if _, err := GetJSON(url, &mirrorList, w.tlsConfig); err != nil { + if _, err := GetJSON(url, &mirrorList, w.httpClient); err != nil { logger.Error("Failed to fetch job status: %s", err.Error()) } From 41e1f263a5394a0fa7e9fdee3b2d41c50f23ddbf Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 13:52:58 +0800 Subject: [PATCH 55/66] refactor(worker): use atomic state to simplify job control --- worker/job.go | 69 ++++++++++++++++++++++++++---------------------- worker/worker.go | 40 +++++++++++++++------------- 2 files changed, 58 insertions(+), 51 deletions(-) diff --git a/worker/job.go b/worker/job.go index de5f79a..924aaf0 100644 --- a/worker/job.go +++ b/worker/job.go @@ -3,6 +3,7 @@ package worker import ( "errors" "fmt" + "sync/atomic" tunasync "github.com/tuna/tunasync/internal" ) @@ -26,22 +27,29 @@ type jobMessage struct { schedule bool } +const ( + // empty state + stateNone uint32 = iota + // ready to run, able to schedule + stateReady + // paused by jobStop + statePaused + // disabled by jobDisable + stateDisabled +) + type mirrorJob struct { - provider mirrorProvider - ctrlChan chan ctrlAction - disabled chan empty - started bool - schedule bool - isDisabled bool + provider mirrorProvider + ctrlChan chan ctrlAction + disabled chan empty + state uint32 } func newMirrorJob(provider mirrorProvider) *mirrorJob { return &mirrorJob{ - provider: provider, - ctrlChan: make(chan ctrlAction, 1), - started: false, - schedule: false, - isDisabled: false, + provider: provider, + ctrlChan: make(chan ctrlAction, 1), + state: stateNone, } } @@ -49,6 +57,14 @@ func (m *mirrorJob) Name() string { return m.provider.Name() } +func (m *mirrorJob) State() uint32 { + return atomic.LoadUint32(&(m.state)) +} + +func (m *mirrorJob) SetState(state uint32) { + atomic.StoreUint32(&(m.state), state) +} + // runMirrorJob is the goroutine where syncing job runs in // arguments: // provider: mirror provider object @@ -61,8 +77,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err m.disabled = make(chan empty) defer func() { close(m.disabled) - m.schedule = false - m.isDisabled = true + m.SetState(stateDisabled) }() provider := m.provider @@ -192,7 +207,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err } for { - if m.started { + if m.State() == stateReady { kill := make(chan empty) jobDone := make(chan empty) go runJob(kill, jobDone) @@ -204,24 +219,21 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err case ctrl := <-m.ctrlChan: switch ctrl { case jobStop: - m.schedule = false - m.started = false + m.SetState(statePaused) close(kill) <-jobDone case jobDisable: - m.schedule = false - m.isDisabled = true - m.started = false + m.SetState(stateDisabled) close(kill) <-jobDone return nil case jobRestart: - m.started = true + m.SetState(stateReady) close(kill) <-jobDone continue case jobStart: - m.started = true + m.SetState(stateReady) goto _wait_for_job default: // TODO: implement this @@ -234,21 +246,14 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err ctrl := <-m.ctrlChan switch ctrl { case jobStop: - m.schedule = false - m.started = false + m.SetState(statePaused) case jobDisable: - m.schedule = false - m.isDisabled = true - m.started = false + m.SetState(stateDisabled) return nil case jobRestart: - m.schedule = true - m.isDisabled = false - m.started = true + m.SetState(stateReady) case jobStart: - m.schedule = true - m.isDisabled = false - m.started = true + m.SetState(stateReady) default: // TODO return nil diff --git a/worker/worker.go b/worker/worker.go index 9cd3f8a..7f31bf8 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -186,32 +186,24 @@ func (w *Worker) makeHTTPServer() { // if job disabled, start them first switch cmd.Cmd { case CmdStart, CmdRestart: - if job.isDisabled { + if job.State() == stateDisabled { go job.Run(w.managerChan, w.semaphore) } } switch cmd.Cmd { case CmdStart: - job.schedule = true - job.isDisabled = false job.ctrlChan <- jobStart case CmdRestart: - job.schedule = true - job.isDisabled = false job.ctrlChan <- jobRestart case CmdStop: // if job is disabled, no goroutine would be there // receiving this signal - if !job.isDisabled { - job.schedule = false - job.isDisabled = false + if job.State() != stateDisabled { w.schedule.Remove(job.Name()) job.ctrlChan <- jobStop } case CmdDisable: - if !job.isDisabled { - job.schedule = false - job.isDisabled = true + if job.State() != stateDisabled { w.schedule.Remove(job.Name()) job.ctrlChan <- jobDisable <-job.disabled @@ -270,15 +262,15 @@ func (w *Worker) runSchedule() { if job, ok := w.jobs[m.Name]; ok { delete(unset, m.Name) switch m.Status { - case Paused: - go job.Run(w.managerChan, w.semaphore) - job.schedule = false - continue case Disabled: - job.schedule = false - job.isDisabled = true + job.SetState(stateDisabled) + continue + case Paused: + job.SetState(statePaused) + go job.Run(w.managerChan, w.semaphore) continue default: + job.SetState(stateReady) go job.Run(w.managerChan, w.semaphore) stime := m.LastUpdate.Add(job.provider.Interval()) logger.Debug("Scheduling job %s @%s", job.Name(), stime.Format("2006-01-02 15:04:05")) @@ -286,8 +278,12 @@ func (w *Worker) runSchedule() { } } } + // some new jobs may be added + // which does not exist in the + // manager's mirror list for name := range unset { job := w.jobs[name] + job.SetState(stateReady) go job.Run(w.managerChan, w.semaphore) w.schedule.AddJob(time.Now(), job) } @@ -297,13 +293,19 @@ func (w *Worker) runSchedule() { case jobMsg := <-w.managerChan: // got status update from job job := w.jobs[jobMsg.name] - if !job.schedule { - logger.Info("Job %s disabled/paused, skip adding new schedule", jobMsg.name) + if job.State() != stateReady { + logger.Info("Job %s state is not ready, skip adding new schedule", jobMsg.name) continue } + // syncing status is only meaningful when job + // is running. If it's paused or disabled + // a sync failure signal would be emitted + // which needs to be ignored w.updateStatus(jobMsg) + // only successful or the final failure msg + // can trigger scheduling if jobMsg.schedule { schedTime := time.Now().Add(job.provider.Interval()) logger.Info( From 924fda6dd89fd0b1b9e723e50ed962f603d4b930 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 16:05:15 +0800 Subject: [PATCH 56/66] feature(worker): use cgroup track job process, so that they can be all-killed --- .travis.yml | 8 ++- tests/worker.conf | 5 ++ worker/cgroup.go | 83 ++++++++++++++++++++++ worker/cgroup_test.go | 108 +++++++++++++++++++++++++++++ worker/cmd_provider.go | 2 +- worker/config.go | 7 ++ worker/provider.go | 12 +++- worker/rsync_provider.go | 2 +- worker/runner.go | 26 ++++--- worker/two_stage_rsync_provider.go | 2 +- worker/worker.go | 12 +++- 11 files changed, 250 insertions(+), 17 deletions(-) create mode 100644 worker/cgroup.go create mode 100644 worker/cgroup_test.go diff --git a/.travis.yml b/.travis.yml index d6a4831..bcba452 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,12 +3,16 @@ go: - 1.6 before_install: - - go get golang.org/x/tools/cmd/cover - - go get -v github.com/mattn/goveralls + - sudo apt-get install cgroup-bin + - go get golang.org/x/tools/cmd/cover + - go get -v github.com/mattn/goveralls os: - linux +before_script: + - sudo cgcreate -t travis -a travis -g cpu:tunasync + script: - ./.testandcover.bash diff --git a/tests/worker.conf b/tests/worker.conf index 0f8f42a..2a27567 100644 --- a/tests/worker.conf +++ b/tests/worker.conf @@ -10,6 +10,11 @@ api_base = "https://localhost:12345" token = "some_token" ca_cert = "rootCA.crt" +[cgroup] +enable = true +base_path = "/sys/fs/cgroup" +group = "tunasync" + [server] hostname = "localhost" listen_addr = "127.0.0.1" diff --git a/worker/cgroup.go b/worker/cgroup.go new file mode 100644 index 0000000..b31eb3f --- /dev/null +++ b/worker/cgroup.go @@ -0,0 +1,83 @@ +package worker + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strconv" + "syscall" + + "golang.org/x/sys/unix" + + "github.com/codeskyblue/go-sh" +) + +type cgroupHook struct { + emptyHook + provider mirrorProvider + basePath string + baseGroup string + created bool +} + +func newCgroupHook(p mirrorProvider, basePath, baseGroup string) *cgroupHook { + if basePath == "" { + basePath = "/sys/fs/cgroup" + } + if baseGroup == "" { + baseGroup = "tunasync" + } + return &cgroupHook{ + provider: p, + basePath: basePath, + baseGroup: baseGroup, + } +} + +func (c *cgroupHook) preExec() error { + c.created = true + return sh.Command("cgcreate", "-g", c.Cgroup()).Run() +} + +func (c *cgroupHook) postExec() error { + err := c.killAll() + if err != nil { + logger.Error("Error killing tasks: %s", err.Error()) + } + + c.created = false + return sh.Command("cgdelete", c.Cgroup()).Run() +} + +func (c *cgroupHook) Cgroup() string { + name := c.provider.Name() + return fmt.Sprintf("cpu:%s/%s", c.baseGroup, name) +} + +func (c *cgroupHook) killAll() error { + if !c.created { + return nil + } + name := c.provider.Name() + taskFile, err := os.Open(filepath.Join(c.basePath, "cpu", c.baseGroup, name, "tasks")) + if err != nil { + return err + } + defer taskFile.Close() + taskList := []int{} + scanner := bufio.NewScanner(taskFile) + for scanner.Scan() { + pid, err := strconv.Atoi(scanner.Text()) + if err != nil { + return err + } + taskList = append(taskList, pid) + } + for _, pid := range taskList { + logger.Debug("Killing process: %d", pid) + unix.Kill(pid, syscall.SIGKILL) + } + + return nil +} diff --git a/worker/cgroup_test.go b/worker/cgroup_test.go new file mode 100644 index 0000000..ba46db1 --- /dev/null +++ b/worker/cgroup_test.go @@ -0,0 +1,108 @@ +package worker + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestCgroup(t *testing.T) { + Convey("Cgroup Should Work", t, func(ctx C) { + tmpDir, err := ioutil.TempDir("", "tunasync") + defer os.RemoveAll(tmpDir) + So(err, ShouldBeNil) + cmdScript := filepath.Join(tmpDir, "cmd.sh") + daemonScript := filepath.Join(tmpDir, "daemon.sh") + tmpFile := filepath.Join(tmpDir, "log_file") + bgPidfile := filepath.Join(tmpDir, "bg.pid") + + c := cmdConfig{ + name: "tuna-cgroup", + upstreamURL: "http://mirrors.tuna.moe/", + command: cmdScript + " " + daemonScript, + workingDir: tmpDir, + logDir: tmpDir, + logFile: tmpFile, + interval: 600 * time.Second, + env: map[string]string{ + "BG_PIDFILE": bgPidfile, + }, + } + cmdScriptContent := `#!/bin/bash +redirect-std() { + [[ -t 0 ]] && exec /dev/null + [[ -t 2 ]] && exec 2>/dev/null +} + +# close all non-std* fds +close-fds() { + eval exec {3..255}\>\&- +} + +# full daemonization of external command with setsid +daemonize() { + ( + redirect-std + cd / + close-fds + exec setsid "$@" + ) & +} + +echo $$ +daemonize $@ +sleep 5 +` + daemonScriptContent := `#!/bin/bash +echo $$ > $BG_PIDFILE +sleep 30 +` + err = ioutil.WriteFile(cmdScript, []byte(cmdScriptContent), 0755) + So(err, ShouldBeNil) + err = ioutil.WriteFile(daemonScript, []byte(daemonScriptContent), 0755) + So(err, ShouldBeNil) + + provider, err := newCmdProvider(c) + So(err, ShouldBeNil) + + cg := newCgroupHook(provider, "/sys/fs/cgroup", "tunasync") + provider.AddHook(cg) + + err = cg.preExec() + So(err, ShouldBeNil) + + go func() { + err = provider.Run() + ctx.So(err, ShouldNotBeNil) + }() + + time.Sleep(1 * time.Second) + // Deamon should be started + daemonPidBytes, err := ioutil.ReadFile(bgPidfile) + So(err, ShouldBeNil) + daemonPid := strings.Trim(string(daemonPidBytes), " \n") + logger.Debug("daemon pid: %s", daemonPid) + procDir := filepath.Join("/proc", daemonPid) + _, err = os.Stat(procDir) + So(err, ShouldBeNil) + + err = provider.Terminate() + So(err, ShouldBeNil) + + // Deamon won't be killed + _, err = os.Stat(procDir) + So(err, ShouldBeNil) + + // Deamon can be killed by cgroup killer + cg.postExec() + _, err = os.Stat(procDir) + So(os.IsNotExist(err), ShouldBeTrue) + + }) +} diff --git a/worker/cmd_provider.go b/worker/cmd_provider.go index 8ed8f9e..7a1d413 100644 --- a/worker/cmd_provider.go +++ b/worker/cmd_provider.go @@ -65,7 +65,7 @@ func (p *cmdProvider) Start() error { for k, v := range p.env { env[k] = v } - p.cmd = newCmdJob(p.command, p.WorkingDir(), env) + p.cmd = newCmdJob(p, p.command, p.WorkingDir(), env) if err := p.prepareLogFile(); err != nil { return err } diff --git a/worker/config.go b/worker/config.go index bb3bd12..9b6c493 100644 --- a/worker/config.go +++ b/worker/config.go @@ -35,6 +35,7 @@ type Config struct { Global globalConfig `toml:"global"` Manager managerConfig `toml:"manager"` Server serverConfig `toml:"server"` + Cgroup cgroupConfig `toml:"cgroup"` Mirrors []mirrorConfig `toml:"mirrors"` } @@ -60,6 +61,12 @@ type serverConfig struct { SSLKey string `toml:"ssl_key"` } +type cgroupConfig struct { + Enable bool `toml:"enable"` + BasePath string `toml:"base_path"` + Group string `toml:"group"` +} + type mirrorConfig struct { Name string `toml:"name"` Provider ProviderEnum `toml:"provider"` diff --git a/worker/provider.go b/worker/provider.go index cf757f0..d83038e 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -33,6 +33,8 @@ type mirrorProvider interface { Terminate() error // job hooks IsRunning() bool + // Cgroup + Cgroup() *cgroupHook AddHook(hook jobHook) Hooks() []jobHook @@ -63,7 +65,8 @@ type baseProvider struct { logFile *os.File - hooks []jobHook + cgroup *cgroupHook + hooks []jobHook } func (p *baseProvider) Name() string { @@ -117,6 +120,9 @@ func (p *baseProvider) LogFile() string { } func (p *baseProvider) AddHook(hook jobHook) { + if cg, ok := hook.(*cgroupHook); ok { + p.cgroup = cg + } p.hooks = append(p.hooks, hook) } @@ -124,6 +130,10 @@ func (p *baseProvider) Hooks() []jobHook { return p.hooks } +func (p *baseProvider) Cgroup() *cgroupHook { + return p.cgroup +} + func (p *baseProvider) prepareLogFile() error { if p.LogFile() == "/dev/null" { p.cmd.SetLogFile(nil) diff --git a/worker/rsync_provider.go b/worker/rsync_provider.go index 49153c9..c3cdefc 100644 --- a/worker/rsync_provider.go +++ b/worker/rsync_provider.go @@ -84,7 +84,7 @@ func (p *rsyncProvider) Start() error { command = append(command, p.options...) command = append(command, p.upstreamURL, p.WorkingDir()) - p.cmd = newCmdJob(command, p.WorkingDir(), env) + p.cmd = newCmdJob(p, command, p.WorkingDir(), env) if err := p.prepareLogFile(); err != nil { return err } diff --git a/worker/runner.go b/worker/runner.go index 49e27bd..8410354 100644 --- a/worker/runner.go +++ b/worker/runner.go @@ -13,7 +13,6 @@ import ( // runner is to run os commands giving command line, env and log file // it's an alternative to python-sh or go-sh -// TODO: cgroup excution var errProcessNotStarted = errors.New("Process Not Started") @@ -23,18 +22,27 @@ type cmdJob struct { env map[string]string logFile *os.File finished chan empty + provider mirrorProvider } -func newCmdJob(cmdAndArgs []string, workingDir string, env map[string]string) *cmdJob { +func newCmdJob(provider mirrorProvider, cmdAndArgs []string, workingDir string, env map[string]string) *cmdJob { var cmd *exec.Cmd - if len(cmdAndArgs) == 1 { - cmd = exec.Command(cmdAndArgs[0]) - } else if len(cmdAndArgs) > 1 { - c := cmdAndArgs[0] - args := cmdAndArgs[1:] + + if provider.Cgroup() != nil { + c := "cgexec" + args := []string{"-g", provider.Cgroup().Cgroup()} + args = append(args, cmdAndArgs...) cmd = exec.Command(c, args...) - } else if len(cmdAndArgs) == 0 { - panic("Command length should be at least 1!") + } else { + if len(cmdAndArgs) == 1 { + cmd = exec.Command(cmdAndArgs[0]) + } else if len(cmdAndArgs) > 1 { + c := cmdAndArgs[0] + args := cmdAndArgs[1:] + cmd = exec.Command(c, args...) + } else if len(cmdAndArgs) == 0 { + panic("Command length should be at least 1!") + } } logger.Debug("Executing command %s at %s", cmdAndArgs[0], workingDir) diff --git a/worker/two_stage_rsync_provider.go b/worker/two_stage_rsync_provider.go index 5a53716..b27cea5 100644 --- a/worker/two_stage_rsync_provider.go +++ b/worker/two_stage_rsync_provider.go @@ -120,7 +120,7 @@ func (p *twoStageRsyncProvider) Run() error { command = append(command, options...) command = append(command, p.upstreamURL, p.WorkingDir()) - p.cmd = newCmdJob(command, p.WorkingDir(), env) + p.cmd = newCmdJob(p, command, p.WorkingDir(), env) if err := p.prepareLogFile(); err != nil { return err } diff --git a/worker/worker.go b/worker/worker.go index 7f31bf8..c780b94 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -152,6 +152,14 @@ func (w *Worker) initProviders() { } provider.AddHook(newLogLimiter(provider)) + + // Add Cgroup Hook + if w.cfg.Cgroup.Enable { + provider.AddHook( + newCgroupHook(provider, w.cfg.Cgroup.BasePath, w.cfg.Cgroup.Group), + ) + } + w.providers[provider.Name()] = provider } @@ -198,13 +206,13 @@ func (w *Worker) makeHTTPServer() { case CmdStop: // if job is disabled, no goroutine would be there // receiving this signal + w.schedule.Remove(job.Name()) if job.State() != stateDisabled { - w.schedule.Remove(job.Name()) job.ctrlChan <- jobStop } case CmdDisable: + w.schedule.Remove(job.Name()) if job.State() != stateDisabled { - w.schedule.Remove(job.Name()) job.ctrlChan <- jobDisable <-job.disabled } From bda7e50d3cd05bb1eaab06da128e3e0c430e1440 Mon Sep 17 00:00:00 2001 From: walkerning Date: Fri, 29 Apr 2016 14:20:22 +0800 Subject: [PATCH 57/66] feature(cmd): add tunasync command line tool --- cmd/tunasync/tunasync.go | 152 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 cmd/tunasync/tunasync.go diff --git a/cmd/tunasync/tunasync.go b/cmd/tunasync/tunasync.go new file mode 100644 index 0000000..afeed56 --- /dev/null +++ b/cmd/tunasync/tunasync.go @@ -0,0 +1,152 @@ +package main + +import ( + "os" + + "github.com/codegangsta/cli" + "gopkg.in/op/go-logging.v1" + + tunasync "github.com/tuna/tunasync/internal" + "github.com/tuna/tunasync/manager" + "github.com/tuna/tunasync/worker" +) + +var logger = logging.MustGetLogger("tunasync-cmd") + +func startManager(c *cli.Context) { + tunasync.InitLogger(c.Bool("verbose"), c.Bool("debug"), c.Bool("with-systemd")) + + cfg, err := manager.LoadConfig(c.String("config"), c) + if err != nil { + logger.Error("Error loading config: %s", err.Error()) + os.Exit(1) + } + + m := manager.GetTUNASyncManager(cfg) + if m == nil { + logger.Error("Error intializing TUNA sync worker.") + os.Exit(1) + } + + logger.Info("Run tunasync manager server.") + m.Run() +} + +func startWorker(c *cli.Context) { + tunasync.InitLogger(c.Bool("verbose"), c.Bool("debug"), c.Bool("with-systemd")) + + cfg, err := worker.LoadConfig(c.String("config")) + if err != nil { + logger.Error("Error loading config: %s", err.Error()) + os.Exit(1) + } + + w := worker.GetTUNASyncWorker(cfg) + if w == nil { + logger.Error("Error intializing TUNA sync worker.") + os.Exit(1) + } + + logger.Info("Run tunasync worker.") + w.Run() +} + +func main() { + app := cli.NewApp() + app.EnableBashCompletion = true + app.Version = "0.1" + app.Commands = []cli.Command{ + { + Name: "manager", + Aliases: []string{"m"}, + Usage: "start the tunasync manager", + Action: startManager, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config, c", + Usage: "Load manager configurations from `FILE`", + }, + + cli.StringFlag{ + Name: "addr", + Usage: "The manager will listen on `ADDR`", + }, + cli.StringFlag{ + Name: "port", + Usage: "The manager will bind to `PORT`", + }, + cli.StringFlag{ + Name: "cert", + Usage: "Use SSL certificate from `FILE`", + }, + cli.StringFlag{ + Name: "key", + Usage: "Use SSL key from `FILE`", + }, + cli.StringFlag{ + Name: "status-file", + Usage: "Write status file to `FILE`", + }, + cli.StringFlag{ + Name: "db-file", + Usage: "Use `FILE` as the database file", + }, + cli.StringFlag{ + Name: "db-type", + Usage: "Use database type `TYPE`", + }, + + cli.BoolFlag{ + Name: "verbose, v", + Usage: "Enable verbose logging", + }, + cli.BoolFlag{ + Name: "debug", + Usage: "Run manager in debug mode", + }, + cli.BoolFlag{ + Name: "with-systemd", + Usage: "Enable systemd-compatible logging", + }, + + cli.StringFlag{ + Name: "pidfile", + Value: "/run/tunasync/tunasync.manager.pid", + Usage: "The pid file of the manager process", + }, + }, + }, + { + Name: "worker", + Aliases: []string{"w"}, + Usage: "start the tunasync worker", + Action: startWorker, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config, c", + Usage: "Load worker configurations from `FILE`", + }, + + cli.BoolFlag{ + Name: "verbose, v", + Usage: "Enable verbose logging", + }, + cli.BoolFlag{ + Name: "debug", + Usage: "Run manager in debug mode", + }, + cli.BoolFlag{ + Name: "with-systemd", + Usage: "Enable systemd-compatible logging", + }, + + cli.StringFlag{ + Name: "pidfile", + Value: "/run/tunasync/tunasync.worker.pid", + Usage: "The pid file of the worker process", + }, + }, + }, + } + app.Run(os.Args) +} From 6e84da0f6abc775515e783eb0ef51462bbb87ad4 Mon Sep 17 00:00:00 2001 From: walkerning Date: Fri, 29 Apr 2016 15:14:02 +0800 Subject: [PATCH 58/66] feature(cmd): add tunasynctl command line tool --- cmd/tunasynctl/tunasynctl.go | 252 +++++++++++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 cmd/tunasynctl/tunasynctl.go diff --git a/cmd/tunasynctl/tunasynctl.go b/cmd/tunasynctl/tunasynctl.go new file mode 100644 index 0000000..b6fe8b1 --- /dev/null +++ b/cmd/tunasynctl/tunasynctl.go @@ -0,0 +1,252 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + + "github.com/codegangsta/cli" + "gopkg.in/op/go-logging.v1" + + tunasync "github.com/tuna/tunasync/internal" +) + +const ( + listJobsPath = "/jobs" + listWorkersPath = "/workers" + cmdPath = "/cmd" +) + +var logger = logging.MustGetLogger("tunasynctl-cmd") + +var baseURL string +var client *http.Client + +func initializeWrapper(handler func(*cli.Context)) func(*cli.Context) { + return func(c *cli.Context) { + err := initialize(c) + if err != nil { + os.Exit(1) + } + handler(c) + } + +} + +func initialize(c *cli.Context) error { + // init logger + tunasync.InitLogger(c.Bool("verbose"), c.Bool("verbose"), false) + + // parse manager server address + baseURL = c.String("manager") + if baseURL == "" { + baseURL = "localhost" + } + managerPort := c.String("port") + if managerPort != "" { + baseURL += ":" + managerPort + } + if c.Bool("no-ssl") { + baseURL = "http://" + baseURL + } else { + baseURL = "https://" + baseURL + } + logger.Info("Use manager address: %s", baseURL) + + // create HTTP client + var err error + client, err = tunasync.CreateHTTPClient(c.String("ca-cert")) + if err != nil { + err = fmt.Errorf("Error initializing HTTP client: %s", err.Error()) + logger.Error(err.Error()) + return err + + } + return nil +} + +func listWorkers(c *cli.Context) { + var workers []tunasync.WorkerStatus + _, err := tunasync.GetJSON(baseURL+listWorkersPath, &workers, client) + if err != nil { + logger.Error("Filed to correctly get informations from manager server: %s", err.Error()) + os.Exit(1) + } + + b, err := json.MarshalIndent(workers, "", " ") + if err != nil { + logger.Error("Error printing out informations: %s", err.Error()) + } + fmt.Print(string(b)) +} + +func listJobs(c *cli.Context) { + // FIXME: there should be an API on manager server side that return MirrorStatus list to tunasynctl + var jobs []tunasync.MirrorStatus + if c.Bool("all") { + _, err := tunasync.GetJSON(baseURL+listJobsPath, &jobs, client) + if err != nil { + logger.Error("Filed to correctly get information of all jobs from manager server: %s", err.Error()) + os.Exit(1) + } + + } else { + args := c.Args() + if len(args) == 0 { + logger.Error("Usage Error: jobs command need at least one arguments or \"--all\" flag.") + os.Exit(1) + } + ans := make(chan []tunasync.MirrorStatus, len(args)) + for _, workerID := range args { + go func(workerID string) { + var workerJobs []tunasync.MirrorStatus + _, err := tunasync.GetJSON(fmt.Sprintf("%s/workers/%s/jobs", baseURL, workerID), &workerJobs, client) + if err != nil { + logger.Error("Filed to correctly get jobs for worker %s: %s", workerID, err.Error()) + } + ans <- workerJobs + }(workerID) + } + for range args { + jobs = append(jobs, <-ans...) + } + } + + b, err := json.MarshalIndent(jobs, "", " ") + if err != nil { + logger.Error("Error printing out informations: %s", err.Error()) + } + fmt.Printf(string(b)) +} + +func cmdJob(cmd tunasync.CmdVerb) func(*cli.Context) { + return func(c *cli.Context) { + var mirrorID string + var argsList []string + if len(c.Args()) == 1 { + mirrorID = c.Args()[0] + } else if len(c.Args()) == 2 { + mirrorID = c.Args()[0] + for _, arg := range strings.Split(c.Args()[1], ",") { + argsList = append(argsList, strings.TrimSpace(arg)) + } + } else { + logger.Error("Usage Error: cmd command receive just 1 required positional argument MIRROR and 1 optional ") + os.Exit(1) + } + + cmd := tunasync.ClientCmd{ + Cmd: cmd, + MirrorID: mirrorID, + WorkerID: c.String("worker"), + Args: argsList, + } + resp, err := tunasync.PostJSON(baseURL+cmdPath, cmd, client) + if err != nil { + logger.Error("Failed to correctly send command: %s", err.Error) + os.Exit(1) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logger.Error("Failed to parse response: %s", err.Error()) + } + + logger.Error("Failed to correctly send command: HTTP status code is not 200: %s", body) + } else { + logger.Info("Succesfully send command") + } + } +} + +func main() { + app := cli.NewApp() + app.EnableBashCompletion = true + app.Version = "0.1" + + commonFlags := []cli.Flag{ + cli.StringFlag{ + Name: "manager, m", + Usage: "The manager server address", + }, + cli.StringFlag{ + Name: "port, p", + Usage: "The manager server port", + }, + cli.StringFlag{ + Name: "ca-cert, c", + Usage: "Trust CA cert `CERT`", + }, + + cli.BoolFlag{ + Name: "no-ssl", + Usage: "Use http rather than https", + }, + cli.BoolFlag{ + Name: "verbose, v", + Usage: "Enable verbosely logging", + }, + } + cmdFlags := []cli.Flag{ + cli.StringFlag{ + Name: "worker, w", + Usage: "Send the command to `WORKER`", + }, + } + + app.Commands = []cli.Command{ + { + Name: "list", + Usage: "List jobs of workers", + Flags: append(commonFlags, + []cli.Flag{ + cli.BoolFlag{ + Name: "all, a", + Usage: "List all jobs of all workers", + }, + }...), + Action: initializeWrapper(listJobs), + }, + { + Name: "workers", + Usage: "List workers", + Flags: commonFlags, + Action: initializeWrapper(listWorkers), + }, + { + Name: "start", + Usage: "Start a job", + Flags: append(commonFlags, cmdFlags...), + Action: initializeWrapper(cmdJob(tunasync.CmdStart)), + }, + { + Name: "stop", + Usage: "Stop a job", + Flags: append(commonFlags, cmdFlags...), + Action: initializeWrapper(cmdJob(tunasync.CmdStop)), + }, + { + Name: "disable", + Usage: "Disable a job", + Flags: append(commonFlags, cmdFlags...), + Action: initializeWrapper(cmdJob(tunasync.CmdDisable)), + }, + { + Name: "restart", + Usage: "Restart a job", + Flags: append(commonFlags, cmdFlags...), + Action: initializeWrapper(cmdJob(tunasync.CmdRestart)), + }, + { + Name: "ping", + Flags: append(commonFlags, cmdFlags...), + Action: initializeWrapper(cmdJob(tunasync.CmdPing)), + }, + } + app.Run(os.Args) +} From f8fd1ae460cae837c47b800186048cca5d4cf5c3 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 20:26:51 +0800 Subject: [PATCH 59/66] style: better logging --- cmd/tunasync/tunasync.go | 7 +++++++ internal/logger.go | 6 +++++- internal/status.go | 24 +++++++++++++++--------- manager/config.go | 2 +- manager/middleware.go | 2 +- manager/server.go | 32 ++++++++++++++++++++++++++++---- tests/manager.conf | 2 +- worker/cgroup.go | 4 ++-- worker/config.go | 2 +- worker/job.go | 12 ++++++------ worker/loglimit_hook.go | 2 +- worker/provider.go | 4 ++-- worker/runner.go | 6 +++--- worker/worker.go | 17 ++++++++--------- 14 files changed, 81 insertions(+), 41 deletions(-) diff --git a/cmd/tunasync/tunasync.go b/cmd/tunasync/tunasync.go index afeed56..dfe45d8 100644 --- a/cmd/tunasync/tunasync.go +++ b/cmd/tunasync/tunasync.go @@ -4,6 +4,7 @@ import ( "os" "github.com/codegangsta/cli" + "github.com/gin-gonic/gin" "gopkg.in/op/go-logging.v1" tunasync "github.com/tuna/tunasync/internal" @@ -21,6 +22,9 @@ func startManager(c *cli.Context) { logger.Error("Error loading config: %s", err.Error()) os.Exit(1) } + if !cfg.Debug { + gin.SetMode(gin.ReleaseMode) + } m := manager.GetTUNASyncManager(cfg) if m == nil { @@ -34,6 +38,9 @@ func startManager(c *cli.Context) { func startWorker(c *cli.Context) { tunasync.InitLogger(c.Bool("verbose"), c.Bool("debug"), c.Bool("with-systemd")) + if !c.Bool("debug") { + gin.SetMode(gin.ReleaseMode) + } cfg, err := worker.LoadConfig(c.String("config")) if err != nil { diff --git a/internal/logger.go b/internal/logger.go index 9ac0632..f733947 100644 --- a/internal/logger.go +++ b/internal/logger.go @@ -12,7 +12,11 @@ func InitLogger(verbose, debug, withSystemd bool) { if withSystemd { fmtString = "[%{level:.6s}] %{message}" } else { - fmtString = "%{color}[%{time:06-01-02 15:04:05}][%{level:.6s}][%{shortfile}]%{color:reset} %{message}" + if debug { + fmtString = "%{color}[%{time:06-01-02 15:04:05}][%{level:.6s}][%{shortfile}]%{color:reset} %{message}" + } else { + fmtString = "%{color}[%{time:06-01-02 15:04:05}][%{level:.6s}]%{color:reset} %{message}" + } } format := logging.MustStringFormatter(fmtString) logging.SetFormatter(format) diff --git a/internal/status.go b/internal/status.go index 8d20a73..79a33ce 100644 --- a/internal/status.go +++ b/internal/status.go @@ -18,24 +18,30 @@ const ( Disabled ) -func (s SyncStatus) MarshalJSON() ([]byte, error) { - var strStatus string +func (s SyncStatus) String() string { switch s { case None: - strStatus = "none" + return "none" case Failed: - strStatus = "failed" + return "failed" case Success: - strStatus = "success" + return "success" case Syncing: - strStatus = "syncing" + return "syncing" case PreSyncing: - strStatus = "pre-syncing" + return "pre-syncing" case Paused: - strStatus = "paused" + return "paused" case Disabled: - strStatus = "disabled" + return "disabled" default: + return "" + } +} + +func (s SyncStatus) MarshalJSON() ([]byte, error) { + strStatus := s.String() + if strStatus == "" { return []byte{}, errors.New("Invalid status value") } diff --git a/manager/config.go b/manager/config.go index ca1f00f..f05a2e8 100644 --- a/manager/config.go +++ b/manager/config.go @@ -41,7 +41,7 @@ func LoadConfig(cfgFile string, c *cli.Context) (*Config, error) { if cfgFile != "" { if _, err := toml.DecodeFile(cfgFile, cfg); err != nil { - logger.Error(err.Error()) + logger.Errorf(err.Error()) return nil, err } } diff --git a/manager/middleware.go b/manager/middleware.go index 67e9266..84dfa1a 100644 --- a/manager/middleware.go +++ b/manager/middleware.go @@ -11,7 +11,7 @@ func contextErrorLogger(c *gin.Context) { errs := c.Errors.ByType(gin.ErrorTypeAny) if len(errs) > 0 { for _, err := range errs { - logger.Error(`"in request "%s %s: %s"`, + logger.Errorf(`"in request "%s %s: %s"`, c.Request.Method, c.Request.URL.Path, err.Error()) } diff --git a/manager/server.go b/manager/server.go index 2d1ada9..30b4a3b 100644 --- a/manager/server.go +++ b/manager/server.go @@ -37,14 +37,19 @@ func GetTUNASyncManager(cfg *Config) *Manager { } s := &Manager{ cfg: cfg, - engine: gin.Default(), adapter: nil, } + s.engine = gin.New() + s.engine.Use(gin.Recovery()) + if cfg.Debug { + s.engine.Use(gin.Logger()) + } + if cfg.Files.CACert != "" { httpClient, err := CreateHTTPClient(cfg.Files.CACert) if err != nil { - logger.Error("Error initializing HTTP client: %s", err.Error()) + logger.Errorf("Error initializing HTTP client: %s", err.Error()) return nil } s.httpClient = httpClient @@ -53,7 +58,7 @@ func GetTUNASyncManager(cfg *Config) *Manager { if cfg.Files.DBFile != "" { adapter, err := makeDBAdapter(cfg.Files.DBType, cfg.Files.DBFile) if err != nil { - logger.Error("Error initializing DB adapter: %s", err.Error()) + logger.Errorf("Error initializing DB adapter: %s", err.Error()) return nil } s.setDBAdapter(adapter) @@ -170,6 +175,8 @@ func (s *Manager) registerWorker(c *gin.Context) { s.returnErrJSON(c, http.StatusInternalServerError, err) return } + + logger.Noticef("Worker <%s> registered", _worker.ID) // create workerCmd channel for this worker c.JSON(http.StatusOK, newWorker) } @@ -210,6 +217,22 @@ func (s *Manager) updateJobOfWorker(c *gin.Context) { status.LastUpdate = curStatus.LastUpdate } + // for logging + switch status.Status { + case Success: + logger.Noticef("Job [%s] @<%s> success", status.Name, status.Worker) + case Failed: + logger.Warningf("Job [%s] @<%s> failed", status.Name, status.Worker) + case Syncing: + logger.Infof("Job [%s] @<%s> starts syncing", status.Name, status.Worker) + case Disabled: + logger.Noticef("Job [%s] @<%s> disabled", status.Name, status.Worker) + case Paused: + logger.Noticef("Job [%s] @<%s> paused", status.Name, status.Worker) + default: + logger.Infof("Job [%s] @<%s> status: %s", status.Name, status.Worker, status.Status) + } + newStatus, err := s.adapter.UpdateMirrorStatus(workerID, mirrorName, status) if err != nil { err := fmt.Errorf("failed to update job %s of worker %s: %s", @@ -228,7 +251,7 @@ func (s *Manager) handleClientCmd(c *gin.Context) { workerID := clientCmd.WorkerID if workerID == "" { // TODO: decide which worker should do this mirror when WorkerID is null string - logger.Error("handleClientCmd case workerID == \" \" not implemented yet") + logger.Errorf("handleClientCmd case workerID == \" \" not implemented yet") c.AbortWithStatus(http.StatusInternalServerError) return } @@ -263,6 +286,7 @@ func (s *Manager) handleClientCmd(c *gin.Context) { s.adapter.UpdateMirrorStatus(clientCmd.WorkerID, clientCmd.MirrorID, curStat) } + logger.Noticef("Posting command '%s %s' to <%s>", clientCmd.Cmd, clientCmd.MirrorID, clientCmd.WorkerID) // post command to worker _, err = PostJSON(workerURL, workerCmd, s.httpClient) if err != nil { diff --git a/tests/manager.conf b/tests/manager.conf index 3f6a45f..0183bde 100644 --- a/tests/manager.conf +++ b/tests/manager.conf @@ -1,4 +1,4 @@ -debug = true +debug = false [server] addr = "127.0.0.1" diff --git a/worker/cgroup.go b/worker/cgroup.go index b31eb3f..f38fc4a 100644 --- a/worker/cgroup.go +++ b/worker/cgroup.go @@ -43,7 +43,7 @@ func (c *cgroupHook) preExec() error { func (c *cgroupHook) postExec() error { err := c.killAll() if err != nil { - logger.Error("Error killing tasks: %s", err.Error()) + logger.Errorf("Error killing tasks: %s", err.Error()) } c.created = false @@ -75,7 +75,7 @@ func (c *cgroupHook) killAll() error { taskList = append(taskList, pid) } for _, pid := range taskList { - logger.Debug("Killing process: %d", pid) + logger.Debugf("Killing process: %d", pid) unix.Kill(pid, syscall.SIGKILL) } diff --git a/worker/config.go b/worker/config.go index 9b6c493..292baa6 100644 --- a/worker/config.go +++ b/worker/config.go @@ -91,7 +91,7 @@ func LoadConfig(cfgFile string) (*Config, error) { cfg := new(Config) if _, err := toml.DecodeFile(cfgFile, cfg); err != nil { - logger.Error(err.Error()) + logger.Errorf(err.Error()) return nil, err } return cfg, nil diff --git a/worker/job.go b/worker/job.go index 924aaf0..9e2afb6 100644 --- a/worker/job.go +++ b/worker/job.go @@ -86,7 +86,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err runHooks := func(Hooks []jobHook, action func(h jobHook) error, hookname string) error { for _, hook := range Hooks { if err := action(hook); err != nil { - logger.Error( + logger.Errorf( "failed at %s hooks for %s: %s", hookname, m.Name(), err.Error(), ) @@ -105,7 +105,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err defer close(jobDone) managerChan <- jobMessage{tunasync.PreSyncing, m.Name(), "", false} - logger.Info("start syncing: %s", m.Name()) + logger.Noticef("start syncing: %s", m.Name()) Hooks := provider.Hooks() rHooks := []jobHook{} @@ -123,7 +123,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err stopASAP := false // stop job as soon as possible if retry > 0 { - logger.Info("retry syncing: %s, retry: %d", m.Name(), retry) + logger.Noticef("retry syncing: %s, retry: %d", m.Name(), retry) } err := runHooks(Hooks, func(h jobHook) error { return h.preExec() }, "pre-exec") if err != nil { @@ -150,7 +150,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err stopASAP = true err := provider.Terminate() if err != nil { - logger.Error("failed to terminate provider %s: %s", m.Name(), err.Error()) + logger.Errorf("failed to terminate provider %s: %s", m.Name(), err.Error()) return err } syncErr = errors.New("killed by manager") @@ -164,7 +164,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err if syncErr == nil { // syncing success - logger.Info("succeeded syncing %s", m.Name()) + logger.Noticef("succeeded syncing %s", m.Name()) managerChan <- jobMessage{tunasync.Success, m.Name(), "", true} // post-success hooks err := runHooks(rHooks, func(h jobHook) error { return h.postSuccess() }, "post-success") @@ -176,7 +176,7 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err } // syncing failed - logger.Warning("failed syncing %s: %s", m.Name(), syncErr.Error()) + logger.Warningf("failed syncing %s: %s", m.Name(), syncErr.Error()) managerChan <- jobMessage{tunasync.Failed, m.Name(), syncErr.Error(), retry == maxRetry-1} // post-fail hooks diff --git a/worker/loglimit_hook.go b/worker/loglimit_hook.go index fdf55e3..69367b3 100644 --- a/worker/loglimit_hook.go +++ b/worker/loglimit_hook.go @@ -30,7 +30,7 @@ func (f fileSlice) Swap(i, j int) { f[i], f[j] = f[j], f[i] } func (f fileSlice) Less(i, j int) bool { return f[i].ModTime().Before(f[j].ModTime()) } func (l *logLimiter) preExec() error { - logger.Debug("executing log limitter for %s", l.provider.Name()) + logger.Debugf("executing log limitter for %s", l.provider.Name()) p := l.provider if p.LogFile() == "/dev/null" { diff --git a/worker/provider.go b/worker/provider.go index d83038e..6686290 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -142,7 +142,7 @@ func (p *baseProvider) prepareLogFile() error { if p.logFile == nil { logFile, err := os.OpenFile(p.LogFile(), os.O_WRONLY|os.O_CREATE, 0644) if err != nil { - logger.Error("Error opening logfile %s: %s", p.LogFile(), err.Error()) + logger.Errorf("Error opening logfile %s: %s", p.LogFile(), err.Error()) return err } p.logFile = logFile @@ -178,7 +178,7 @@ func (p *baseProvider) Wait() error { } func (p *baseProvider) Terminate() error { - logger.Debug("terminating provider: %s", p.Name()) + logger.Debugf("terminating provider: %s", p.Name()) if !p.IsRunning() { return nil } diff --git a/worker/runner.go b/worker/runner.go index 8410354..04fc1fb 100644 --- a/worker/runner.go +++ b/worker/runner.go @@ -45,11 +45,11 @@ func newCmdJob(provider mirrorProvider, cmdAndArgs []string, workingDir string, } } - logger.Debug("Executing command %s at %s", cmdAndArgs[0], workingDir) + logger.Debugf("Executing command %s at %s", cmdAndArgs[0], workingDir) if _, err := os.Stat(workingDir); os.IsNotExist(err) { - logger.Debug("Making dir %s", workingDir) + logger.Debugf("Making dir %s", workingDir) if err = os.MkdirAll(workingDir, 0755); err != nil { - logger.Error("Error making dir %s", workingDir) + logger.Errorf("Error making dir %s", workingDir) } } diff --git a/worker/worker.go b/worker/worker.go index c780b94..67039f8 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -49,7 +49,7 @@ func GetTUNASyncWorker(cfg *Config) *Worker { if cfg.Manager.CACert != "" { httpClient, err := CreateHTTPClient(cfg.Manager.CACert) if err != nil { - logger.Error("Error initializing HTTP client: %s", err.Error()) + logger.Errorf("Error initializing HTTP client: %s", err.Error()) return nil } w.httpClient = httpClient @@ -190,7 +190,7 @@ func (w *Worker) makeHTTPServer() { c.JSON(http.StatusNotFound, gin.H{"msg": fmt.Sprintf("Mirror ``%s'' not found", cmd.MirrorID)}) return } - logger.Info("Received command: %v", cmd) + logger.Noticef("Received command: %v", cmd) // if job disabled, start them first switch cmd.Cmd { case CmdStart, CmdRestart: @@ -225,7 +225,6 @@ func (w *Worker) makeHTTPServer() { c.JSON(http.StatusOK, gin.H{"msg": "OK"}) }) - w.httpEngine = s } @@ -281,7 +280,7 @@ func (w *Worker) runSchedule() { job.SetState(stateReady) go job.Run(w.managerChan, w.semaphore) stime := m.LastUpdate.Add(job.provider.Interval()) - logger.Debug("Scheduling job %s @%s", job.Name(), stime.Format("2006-01-02 15:04:05")) + logger.Debugf("Scheduling job %s @%s", job.Name(), stime.Format("2006-01-02 15:04:05")) w.schedule.AddJob(stime, job) } } @@ -302,7 +301,7 @@ func (w *Worker) runSchedule() { // got status update from job job := w.jobs[jobMsg.name] if job.State() != stateReady { - logger.Info("Job %s state is not ready, skip adding new schedule", jobMsg.name) + logger.Infof("Job %s state is not ready, skip adding new schedule", jobMsg.name) continue } @@ -316,7 +315,7 @@ func (w *Worker) runSchedule() { // can trigger scheduling if jobMsg.schedule { schedTime := time.Now().Add(job.provider.Interval()) - logger.Info( + logger.Noticef( "Next scheduled time for %s: %s", job.Name(), schedTime.Format("2006-01-02 15:04:05"), @@ -362,7 +361,7 @@ func (w *Worker) registorWorker() { } if _, err := PostJSON(url, msg, w.httpClient); err != nil { - logger.Error("Failed to register worker") + logger.Errorf("Failed to register worker") } } @@ -385,7 +384,7 @@ func (w *Worker) updateStatus(jobMsg jobMessage) { } if _, err := PostJSON(url, smsg, w.httpClient); err != nil { - logger.Error("Failed to update mirror(%s) status: %s", jobMsg.name, err.Error()) + logger.Errorf("Failed to update mirror(%s) status: %s", jobMsg.name, err.Error()) } } @@ -399,7 +398,7 @@ func (w *Worker) fetchJobStatus() []MirrorStatus { ) if _, err := GetJSON(url, &mirrorList, w.httpClient); err != nil { - logger.Error("Failed to fetch job status: %s", err.Error()) + logger.Errorf("Failed to fetch job status: %s", err.Error()) } return mirrorList From a644294bd7dfb60887d356f947b16da9782d635b Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 20:36:13 +0800 Subject: [PATCH 60/66] style: better logging for tunasynctl --- cmd/tunasynctl/tunasynctl.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cmd/tunasynctl/tunasynctl.go b/cmd/tunasynctl/tunasynctl.go index b6fe8b1..f4ddc54 100644 --- a/cmd/tunasynctl/tunasynctl.go +++ b/cmd/tunasynctl/tunasynctl.go @@ -33,7 +33,6 @@ func initializeWrapper(handler func(*cli.Context)) func(*cli.Context) { } handler(c) } - } func initialize(c *cli.Context) error { @@ -54,7 +53,7 @@ func initialize(c *cli.Context) error { } else { baseURL = "https://" + baseURL } - logger.Info("Use manager address: %s", baseURL) + logger.Infof("Use manager address: %s", baseURL) // create HTTP client var err error @@ -72,13 +71,13 @@ func listWorkers(c *cli.Context) { var workers []tunasync.WorkerStatus _, err := tunasync.GetJSON(baseURL+listWorkersPath, &workers, client) if err != nil { - logger.Error("Filed to correctly get informations from manager server: %s", err.Error()) + logger.Errorf("Filed to correctly get informations from manager server: %s", err.Error()) os.Exit(1) } b, err := json.MarshalIndent(workers, "", " ") if err != nil { - logger.Error("Error printing out informations: %s", err.Error()) + logger.Errorf("Error printing out informations: %s", err.Error()) } fmt.Print(string(b)) } @@ -89,7 +88,7 @@ func listJobs(c *cli.Context) { if c.Bool("all") { _, err := tunasync.GetJSON(baseURL+listJobsPath, &jobs, client) if err != nil { - logger.Error("Filed to correctly get information of all jobs from manager server: %s", err.Error()) + logger.Errorf("Filed to correctly get information of all jobs from manager server: %s", err.Error()) os.Exit(1) } @@ -105,7 +104,7 @@ func listJobs(c *cli.Context) { var workerJobs []tunasync.MirrorStatus _, err := tunasync.GetJSON(fmt.Sprintf("%s/workers/%s/jobs", baseURL, workerID), &workerJobs, client) if err != nil { - logger.Error("Filed to correctly get jobs for worker %s: %s", workerID, err.Error()) + logger.Errorf("Filed to correctly get jobs for worker %s: %s", workerID, err.Error()) } ans <- workerJobs }(workerID) @@ -117,7 +116,7 @@ func listJobs(c *cli.Context) { b, err := json.MarshalIndent(jobs, "", " ") if err != nil { - logger.Error("Error printing out informations: %s", err.Error()) + logger.Errorf("Error printing out informations: %s", err.Error()) } fmt.Printf(string(b)) } @@ -146,7 +145,7 @@ func cmdJob(cmd tunasync.CmdVerb) func(*cli.Context) { } resp, err := tunasync.PostJSON(baseURL+cmdPath, cmd, client) if err != nil { - logger.Error("Failed to correctly send command: %s", err.Error) + logger.Errorf("Failed to correctly send command: %s", err.Error()) os.Exit(1) } defer resp.Body.Close() @@ -154,10 +153,10 @@ func cmdJob(cmd tunasync.CmdVerb) func(*cli.Context) { if resp.StatusCode != http.StatusOK { body, err := ioutil.ReadAll(resp.Body) if err != nil { - logger.Error("Failed to parse response: %s", err.Error()) + logger.Errorf("Failed to parse response: %s", err.Error()) } - logger.Error("Failed to correctly send command: HTTP status code is not 200: %s", body) + logger.Errorf("Failed to correctly send command: HTTP status code is not 200: %s", body) } else { logger.Info("Succesfully send command") } From bd423eec4ea5b315b88a8c340cd2c1c08c200ea7 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 21:47:42 +0800 Subject: [PATCH 61/66] feature(worker): added exec_on_success and exec_on_failure option and hooks --- worker/config.go | 3 ++ worker/config_test.go | 8 +++ worker/exec_post_hook.go | 96 +++++++++++++++++++++++++++++++++ worker/exec_post_test.go | 112 +++++++++++++++++++++++++++++++++++++++ worker/worker.go | 19 +++++++ 5 files changed, 238 insertions(+) create mode 100644 worker/exec_post_hook.go create mode 100644 worker/exec_post_test.go diff --git a/worker/config.go b/worker/config.go index 292baa6..96f4d28 100644 --- a/worker/config.go +++ b/worker/config.go @@ -76,6 +76,9 @@ type mirrorConfig struct { LogDir string `toml:"log_dir"` Env map[string]string `toml:"env"` + ExecOnSuccess string `toml:"exec_on_success"` + ExecOnFailure string `toml:"exec_on_failure"` + Command string `toml:"command"` UseIPv6 bool `toml:"use_ipv6"` ExcludeFile string `toml:"exclude_file"` diff --git a/worker/config_test.go b/worker/config_test.go index dbebcb7..9c5b6d1 100644 --- a/worker/config_test.go +++ b/worker/config_test.go @@ -34,6 +34,7 @@ provider = "command" upstream = "https://aosp.google.com/" interval = 720 mirror_dir = "/data/git/AOSP" +exec_on_success = "bash -c 'echo ${TUNASYNC_JOB_EXIT_STATUS} > ${TUNASYNC_WORKING_DIR}/exit_status'" [mirrors.env] REPO = "/usr/local/bin/aosp-repo" @@ -51,6 +52,7 @@ provider = "rsync" upstream = "rsync://ftp.fedoraproject.org/fedora/" use_ipv6 = true exclude_file = "/etc/tunasync.d/fedora-exclude.txt" +exec_on_failure = "bash -c 'echo ${TUNASYNC_JOB_EXIT_STATUS} > ${TUNASYNC_WORKING_DIR}/exit_status'" ` Convey("When giving invalid file", t, func() { @@ -123,6 +125,12 @@ exclude_file = "/etc/tunasync.d/fedora-exclude.txt" So(p.LogFile(), ShouldEqual, "/var/log/tunasync/AOSP/latest.log") _, ok := p.(*cmdProvider) So(ok, ShouldBeTrue) + for _, hook := range p.Hooks() { + switch h := hook.(type) { + case *execPostHook: + So(h.command, ShouldResemble, []string{"bash", "-c", `echo ${TUNASYNC_JOB_EXIT_STATUS} > ${TUNASYNC_WORKING_DIR}/exit_status`}) + } + } p = w.providers["debian"] So(p.Name(), ShouldEqual, "debian") diff --git a/worker/exec_post_hook.go b/worker/exec_post_hook.go new file mode 100644 index 0000000..16e5d16 --- /dev/null +++ b/worker/exec_post_hook.go @@ -0,0 +1,96 @@ +package worker + +import ( + "errors" + "fmt" + + "github.com/anmitsu/go-shlex" + "github.com/codeskyblue/go-sh" +) + +// hook to execute command after syncing +// typically setting timestamp, etc. + +const ( + execOnSuccess uint8 = iota + execOnFailure +) + +type execPostHook struct { + emptyHook + provider mirrorProvider + + // exec on success or on failure + execOn uint8 + // command + command []string +} + +func newExecPostHook(provider mirrorProvider, execOn uint8, command string) (*execPostHook, error) { + cmd, err := shlex.Split(command, true) + if err != nil { + // logger.Errorf("Failed to create exec-post-hook for command: %s", command) + return nil, err + } + if execOn != execOnSuccess && execOn != execOnFailure { + return nil, fmt.Errorf("Invalid option for exec-on: %d", execOn) + } + + return &execPostHook{ + provider: provider, + execOn: execOn, + command: cmd, + }, nil +} + +func (h *execPostHook) postSuccess() error { + if h.execOn == execOnSuccess { + return h.Do() + } + return nil +} + +func (h *execPostHook) postFail() error { + if h.execOn == execOnFailure { + return h.Do() + } + return nil +} + +func (h *execPostHook) Do() error { + p := h.provider + + exitStatus := "" + if h.execOn == execOnSuccess { + exitStatus = "success" + } else { + exitStatus = "failure" + } + + env := map[string]string{ + "TUNASYNC_MIRROR_NAME": p.Name(), + "TUNASYNC_WORKING_DIR": p.WorkingDir(), + "TUNASYNC_UPSTREAM_URL": p.Upstream(), + "TUNASYNC_LOG_FILE": p.LogFile(), + "TUNASYNC_JOB_EXIT_STATUS": exitStatus, + } + + session := sh.NewSession() + for k, v := range env { + session.SetEnv(k, v) + } + + var cmd string + args := []interface{}{} + if len(h.command) == 1 { + cmd = h.command[0] + } else if len(h.command) > 1 { + cmd = h.command[0] + for _, arg := range h.command[1:] { + args = append(args, arg) + } + } else { + return errors.New("Invalid Command") + } + return session.Command(cmd, args...).Run() +} diff --git a/worker/exec_post_test.go b/worker/exec_post_test.go new file mode 100644 index 0000000..e2f7efc --- /dev/null +++ b/worker/exec_post_test.go @@ -0,0 +1,112 @@ +package worker + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" + . "github.com/tuna/tunasync/internal" +) + +func TestExecPost(t *testing.T) { + Convey("ExecPost should work", t, func(ctx C) { + tmpDir, err := ioutil.TempDir("", "tunasync") + defer os.RemoveAll(tmpDir) + So(err, ShouldBeNil) + scriptFile := filepath.Join(tmpDir, "cmd.sh") + + c := cmdConfig{ + name: "tuna-exec-post", + upstreamURL: "http://mirrors.tuna.moe/", + command: scriptFile, + workingDir: tmpDir, + logDir: tmpDir, + logFile: filepath.Join(tmpDir, "latest.log"), + interval: 600 * time.Second, + } + + provider, err := newCmdProvider(c) + So(err, ShouldBeNil) + + Convey("On success", func() { + hook, err := newExecPostHook(provider, execOnSuccess, "bash -c 'echo ${TUNASYNC_JOB_EXIT_STATUS} > ${TUNASYNC_WORKING_DIR}/exit_status'") + So(err, ShouldBeNil) + provider.AddHook(hook) + managerChan := make(chan jobMessage) + semaphore := make(chan empty, 1) + job := newMirrorJob(provider) + + scriptContent := `#!/bin/bash +echo $TUNASYNC_WORKING_DIR +echo $TUNASYNC_MIRROR_NAME +echo $TUNASYNC_UPSTREAM_URL +echo $TUNASYNC_LOG_FILE + ` + + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + + go job.Run(managerChan, semaphore) + job.ctrlChan <- jobStart + msg := <-managerChan + So(msg.status, ShouldEqual, PreSyncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Syncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Success) + + time.Sleep(200 * time.Millisecond) + job.ctrlChan <- jobDisable + <-job.disabled + + expectedOutput := "success\n" + + outputContent, err := ioutil.ReadFile(filepath.Join(provider.WorkingDir(), "exit_status")) + So(err, ShouldBeNil) + So(string(outputContent), ShouldEqual, expectedOutput) + }) + + Convey("On failure", func() { + hook, err := newExecPostHook(provider, execOnFailure, "bash -c 'echo ${TUNASYNC_JOB_EXIT_STATUS} > ${TUNASYNC_WORKING_DIR}/exit_status'") + So(err, ShouldBeNil) + provider.AddHook(hook) + managerChan := make(chan jobMessage) + semaphore := make(chan empty, 1) + job := newMirrorJob(provider) + + scriptContent := `#!/bin/bash +echo $TUNASYNC_WORKING_DIR +echo $TUNASYNC_MIRROR_NAME +echo $TUNASYNC_UPSTREAM_URL +echo $TUNASYNC_LOG_FILE +sleep 5 +exit 1 + ` + + err = ioutil.WriteFile(scriptFile, []byte(scriptContent), 0755) + So(err, ShouldBeNil) + + go job.Run(managerChan, semaphore) + job.ctrlChan <- jobStart + msg := <-managerChan + So(msg.status, ShouldEqual, PreSyncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Syncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Failed) + + time.Sleep(200 * time.Millisecond) + job.ctrlChan <- jobDisable + <-job.disabled + + expectedOutput := "failure\n" + + outputContent, err := ioutil.ReadFile(filepath.Join(provider.WorkingDir(), "exit_status")) + So(err, ShouldBeNil) + So(string(outputContent), ShouldEqual, expectedOutput) + }) + }) +} diff --git a/worker/worker.go b/worker/worker.go index 67039f8..64d0bbf 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -160,6 +160,25 @@ func (w *Worker) initProviders() { ) } + // ExecOnSuccess hook + if mirror.ExecOnSuccess != "" { + h, err := newExecPostHook(provider, execOnSuccess, mirror.ExecOnSuccess) + if err != nil { + logger.Errorf("Error initializing mirror %s: %s", mirror.Name, err.Error()) + panic(err) + } + provider.AddHook(h) + } + // ExecOnFailure hook + if mirror.ExecOnFailure != "" { + h, err := newExecPostHook(provider, execOnFailure, mirror.ExecOnFailure) + if err != nil { + logger.Errorf("Error initializing mirror %s: %s", mirror.Name, err.Error()) + panic(err) + } + provider.AddHook(h) + } + w.providers[provider.Name()] = provider } From 4b52b9b53bd472530c99c7208dadd65929958541 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 21:48:46 +0800 Subject: [PATCH 62/66] refactor: goodbye, python2.7 --- tunasync/__init__.py | 4 - tunasync/btrfs_snapshot.py | 62 -------- tunasync/clt_server.py | 57 -------- tunasync/exec_pre_post.py | 36 ----- tunasync/hook.py | 19 --- tunasync/jobs.py | 135 ----------------- tunasync/loglimit.py | 88 ------------ tunasync/mirror_config.py | 156 -------------------- tunasync/mirror_provider.py | 226 ----------------------------- tunasync/status_manager.py | 123 ---------------- tunasync/tunasync.py | 279 ------------------------------------ 11 files changed, 1185 deletions(-) delete mode 100644 tunasync/__init__.py delete mode 100644 tunasync/btrfs_snapshot.py delete mode 100644 tunasync/clt_server.py delete mode 100644 tunasync/exec_pre_post.py delete mode 100644 tunasync/hook.py delete mode 100644 tunasync/jobs.py delete mode 100644 tunasync/loglimit.py delete mode 100644 tunasync/mirror_config.py delete mode 100644 tunasync/mirror_provider.py delete mode 100644 tunasync/status_manager.py delete mode 100644 tunasync/tunasync.py diff --git a/tunasync/__init__.py b/tunasync/__init__.py deleted file mode 100644 index c869f43..0000000 --- a/tunasync/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -from .tunasync import TUNASync -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/btrfs_snapshot.py b/tunasync/btrfs_snapshot.py deleted file mode 100644 index 163e0c7..0000000 --- a/tunasync/btrfs_snapshot.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import sh -import os -from datetime import datetime -from .hook import JobHook - - -class BtrfsVolumeError(Exception): - pass - - -class BtrfsHook(JobHook): - - def __init__(self, service_dir, working_dir, gc_dir): - self.service_dir = service_dir - self.working_dir = working_dir - self.gc_dir = gc_dir - - def before_job(self, ctx={}, *args, **kwargs): - self._create_working_snapshot() - ctx['current_dir'] = self.working_dir - - def after_job(self, status=None, ctx={}, *args, **kwargs): - if status == "success": - self._commit_changes() - ctx['current_dir'] = self.service_dir - - def _ensure_subvolume(self): - # print(self.service_dir) - try: - ret = sh.btrfs("subvolume", "show", self.service_dir) - except Exception, e: - print(e) - raise BtrfsVolumeError("Invalid subvolume") - - if ret.stderr != '': - raise BtrfsVolumeError("Invalid subvolume") - - def _create_working_snapshot(self): - self._ensure_subvolume() - if os.path.exists(self.working_dir): - print("Warning: working dir existed, are you sure no rsync job is running?") - else: - # print("btrfs subvolume snapshot {} {}".format(self.service_dir, self.working_dir)) - sh.btrfs("subvolume", "snapshot", self.service_dir, self.working_dir) - - def _commit_changes(self): - self._ensure_subvolume() - self._ensure_subvolume() - gc_dir = self.gc_dir.format(timestamp=datetime.now().strftime("%s")) - - out = sh.mv(self.service_dir, gc_dir) - assert out.exit_code == 0 and out.stderr == "" - out = sh.mv(self.working_dir, self.service_dir) - assert out.exit_code == 0 and out.stderr == "" - # print("btrfs subvolume delete {}".format(self.tmp_dir)) - # sh.sleep(3) - # out = sh.btrfs("subvolume", "delete", self.tmp_dir) - # assert out.exit_code == 0 and out.stderr == "" - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/clt_server.py b/tunasync/clt_server.py deleted file mode 100644 index 7a815f8..0000000 --- a/tunasync/clt_server.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import socket -import os -import json -import struct - - -class ControlServer(object): - - valid_commands = set(( - "start", "stop", "restart", "status", "log", - )) - - def __init__(self, address, mgr_chan, cld_chan): - self.address = address - self.mgr_chan = mgr_chan - self.cld_chan = cld_chan - try: - os.unlink(self.address) - except OSError: - if os.path.exists(self.address): - raise Exception("file exists: {}".format(self.address)) - self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self.sock.bind(self.address) - os.chmod(address, 0o700) - - print("Control Server listening on: {}".format(self.address)) - self.sock.listen(1) - - def serve_forever(self): - while 1: - conn, _ = self.sock.accept() - - try: - length = struct.unpack('!H', conn.recv(2))[0] - content = conn.recv(length) - cmd = json.loads(content) - if cmd['cmd'] not in self.valid_commands: - raise Exception("Invalid Command") - self.mgr_chan.put(("CMD", (cmd['cmd'], cmd['target'], cmd["kwargs"]))) - except Exception as e: - print(e) - res = "Invalid Command" - else: - res = self.cld_chan.get() - - conn.sendall(struct.pack('!H', len(res))) - conn.sendall(res) - conn.close() - - -def run_control_server(address, mgr_chan, cld_chan): - cs = ControlServer(address, mgr_chan, cld_chan) - cs.serve_forever() - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/exec_pre_post.py b/tunasync/exec_pre_post.py deleted file mode 100644 index 1ade5ad..0000000 --- a/tunasync/exec_pre_post.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import os -import sh -import shlex -from .hook import JobHook - - -class CmdExecHook(JobHook): - POST_SYNC = "post_sync" - PRE_SYNC = "pre_sync" - - def __init__(self, command, exec_at=POST_SYNC): - self.command = shlex.split(command) - if exec_at == self.POST_SYNC: - self.before_job = self._keep_calm - self.after_job = self._exec - elif exec_at == self.PRE_SYNC: - self.before_job = self._exec - self.after_job = self._keep_calm - - def _keep_calm(self, ctx={}, **kwargs): - pass - - def _exec(self, ctx={}, **kwargs): - new_env = os.environ.copy() - new_env["TUNASYNC_MIRROR_NAME"] = ctx["mirror_name"] - new_env["TUNASYNC_WORKING_DIR"] = ctx["current_dir"] - new_env["TUNASYNC_JOB_EXIT_STATUS"] = kwargs.get("status", "") - - _cmd = self.command[0] - _args = [] if len(self.command) == 1 else self.command[1:] - cmd = sh.Command(_cmd) - cmd(*_args, _env=new_env) - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/hook.py b/tunasync/hook.py deleted file mode 100644 index 3f31c30..0000000 --- a/tunasync/hook.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- - - -class JobHook(object): - - def before_job(self, *args, **kwargs): - raise NotImplementedError("") - - def after_job(self, *args, **kwargs): - raise NotImplementedError("") - - def before_exec(self, *args, **kwargs): - pass - - def after_exec(self, *args, **kwargs): - pass - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/jobs.py b/tunasync/jobs.py deleted file mode 100644 index e45d041..0000000 --- a/tunasync/jobs.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import sh -import sys -from setproctitle import setproctitle -import signal -import Queue -import traceback - - -def run_job(sema, child_q, manager_q, provider, **settings): - aquired = False - setproctitle("tunasync-{}".format(provider.name)) - - def before_quit(*args): - provider.terminate() - if aquired: - print("{} release semaphore".format(provider.name)) - sema.release() - sys.exit(0) - - def sleep_wait(timeout): - try: - msg = child_q.get(timeout=timeout) - if msg == "terminate": - manager_q.put(("CONFIG_ACK", (provider.name, "QUIT"))) - return True - except Queue.Empty: - return False - - signal.signal(signal.SIGTERM, before_quit) - - if provider.delay > 0: - if sleep_wait(provider.delay): - return - - max_retry = settings.get("max_retry", 1) - - def _real_run(idx=0, stage="job_hook", ctx=None): - """\ - 4 stages: - 0 -> job_hook, 1 -> set_retry, 2 -> exec_hook, 3 -> exec - """ - - assert(ctx is not None) - - if stage == "exec": - # exec_job - try: - provider.run(ctx=ctx) - provider.wait() - except sh.ErrorReturnCode: - status = "fail" - else: - status = "success" - return status - - elif stage == "set_retry": - # enter stage 3 with retry - for retry in range(max_retry): - status = "syncing" - manager_q.put(("UPDATE", (provider.name, status, ctx))) - print("start syncing {}, retry: {}".format(provider.name, retry)) - status = _real_run(idx=0, stage="exec_hook", ctx=ctx) - if status == "success": - break - return status - - # job_hooks - elif stage == "job_hook": - if idx == len(provider.hooks): - return _real_run(idx=idx, stage="set_retry", ctx=ctx) - hook = provider.hooks[idx] - hook_before, hook_after = hook.before_job, hook.after_job - status = "pre-syncing" - - elif stage == "exec_hook": - if idx == len(provider.hooks): - return _real_run(idx=idx, stage="exec", ctx=ctx) - hook = provider.hooks[idx] - hook_before, hook_after = hook.before_exec, hook.after_exec - status = "syncing" - - try: - # print("%s run before_%s, %d" % (provider.name, stage, idx)) - hook_before(provider=provider, ctx=ctx) - status = _real_run(idx=idx+1, stage=stage, ctx=ctx) - except Exception: - traceback.print_exc() - status = "fail" - finally: - # print("%s run after_%s, %d" % (provider.name, stage, idx)) - # job may break when syncing - if status != "success": - status = "fail" - try: - hook_after(provider=provider, status=status, ctx=ctx) - except Exception: - traceback.print_exc() - - return status - - while 1: - try: - sema.acquire(True) - except: - break - aquired = True - - ctx = {} # put context info in it - ctx['current_dir'] = provider.local_dir - ctx['mirror_name'] = provider.name - status = "pre-syncing" - manager_q.put(("UPDATE", (provider.name, status, ctx))) - - try: - status = _real_run(idx=0, stage="job_hook", ctx=ctx) - except Exception: - traceback.print_exc() - status = "fail" - finally: - sema.release() - aquired = False - - print("syncing {} finished, sleep {} minutes for the next turn".format( - provider.name, provider.interval - )) - - manager_q.put(("UPDATE", (provider.name, status, ctx))) - - if sleep_wait(timeout=provider.interval * 60): - break - - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/loglimit.py b/tunasync/loglimit.py deleted file mode 100644 index 053c63a..0000000 --- a/tunasync/loglimit.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import sh -import os -from .hook import JobHook -from datetime import datetime - - -class LogLimitHook(JobHook): - - def __init__(self, limit=10): - self.limit = limit - - def before_job(self, *args, **kwargs): - pass - - def after_job(self, *args, **kwargs): - pass - - def before_exec(self, provider, ctx={}, *args, **kwargs): - log_dir = provider.log_dir - self.ensure_log_dir(log_dir) - log_file = provider.log_file.format( - date=datetime.now().strftime("%Y-%m-%d_%H-%M")) - ctx['log_file'] = log_file - if log_file == "/dev/null": - return - - log_link = os.path.join(log_dir, "latest") - ctx['log_link'] = log_link - - lfiles = [os.path.join(log_dir, lfile) - for lfile in os.listdir(log_dir) - if lfile.startswith(provider.name)] - - lfiles_set = set(lfiles) - # sort to get the newest 10 files - lfiles_ts = sorted( - [(os.path.getmtime(lfile), lfile) for lfile in lfiles], - key=lambda x: x[0], - reverse=True) - lfiles_keep = set([x[1] for x in lfiles_ts[:self.limit]]) - lfiles_rm = lfiles_set - lfiles_keep - # remove old files - for lfile in lfiles_rm: - try: - sh.rm(lfile) - except: - pass - - # create a soft link - self.create_link(log_link, log_file) - - def after_exec(self, status=None, ctx={}, *args, **kwargs): - log_file = ctx.get('log_file', None) - log_link = ctx.get('log_link', None) - if log_file == "/dev/null": - return - if status == "fail": - log_file_save = log_file + ".fail" - try: - sh.mv(log_file, log_file_save) - except: - pass - self.create_link(log_link, log_file_save) - - def ensure_log_dir(self, log_dir): - if not os.path.exists(log_dir): - sh.mkdir("-p", log_dir) - - def create_link(self, log_link, log_file): - if log_link == log_file: - return - if not (log_link and log_file): - return - - if os.path.lexists(log_link): - try: - sh.rm(log_link) - except: - return - try: - sh.ln('-s', log_file, log_link) - except: - return - - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/mirror_config.py b/tunasync/mirror_config.py deleted file mode 100644 index 8f57198..0000000 --- a/tunasync/mirror_config.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import os -from datetime import datetime -from .mirror_provider import RsyncProvider, TwoStageRsyncProvider, ShellProvider -from .btrfs_snapshot import BtrfsHook -from .loglimit import LogLimitHook -from .exec_pre_post import CmdExecHook - - -class MirrorConfig(object): - - _valid_providers = set(("rsync", "two-stage-rsync", "shell", )) - - def __init__(self, parent, options): - self._parent = parent - self._popt = self._parent._settings - self.options = dict(options.items()) # copy - self._validate() - - def _validate(self): - provider = self.options.get("provider", None) - assert provider in self._valid_providers - - if provider == "rsync": - assert "upstream" in self.options - - elif provider == "shell": - assert "command" in self.options - - local_dir_tmpl = self.options.get( - "local_dir", self._popt["global"]["local_dir"]) - - self.options["local_dir"] = local_dir_tmpl.format( - mirror_root=self._popt["global"]["mirror_root"], - mirror_name=self.name, - ) - - if "interval" not in self.options: - self.options["interval"] = self._popt["global"]["interval"] - - assert isinstance(self.options["interval"], int) - - log_dir = self.options.get( - "log_dir", self._popt["global"]["log_dir"]) - if "log_file" not in self.options: - self.options["log_file"] = os.path.join( - log_dir, self.name, self.name + "_{date}.log") - - self.log_dir = os.path.dirname(self.log_file) - - if "use_btrfs" not in self.options: - self.options["use_btrfs"] = self._parent.use_btrfs - assert self.options["use_btrfs"] in (True, False) - - if "env" in self.options: - assert isinstance(self.options["env"], dict) - - def __getattr__(self, key): - if key in self.__dict__: - return self.__dict__[key] - else: - return self.__dict__["options"].get(key, None) - - def to_provider(self, hooks=[], no_delay=False): - - kwargs = { - 'name': self.name, - 'upstream_url': self.upstream, - 'local_dir': self.local_dir, - 'log_dir': self.log_dir, - 'log_file': self.log_file, - 'interval': self.interval, - 'env': self.env, - 'hooks': hooks, - } - - if self.provider == "rsync": - kwargs.update({ - 'useIPv6': self.use_ipv6, - 'password': self.password, - 'exclude_file': self.exclude_file, - }) - provider = RsyncProvider(**kwargs) - - elif self.provider == "two-stage-rsync": - kwargs.update({ - 'useIPv6': self.use_ipv6, - 'password': self.password, - 'exclude_file': self.exclude_file, - }) - provider = TwoStageRsyncProvider(**kwargs) - provider.set_stage1_profile(self.stage1_profile) - - elif self.options["provider"] == "shell": - kwargs.update({ - 'command': self.command, - 'log_stdout': self.options.get("log_stdout", True), - }) - - provider = ShellProvider(**kwargs) - - if not no_delay: - sm = self._parent.status_manager - last_update = sm.get_info(self.name, 'last_update') - if last_update not in (None, '-'): - last_update = datetime.strptime( - last_update, '%Y-%m-%d %H:%M:%S') - delay = int(last_update.strftime("%s")) \ - + self.interval * 60 - int(datetime.now().strftime("%s")) - if delay < 0: - delay = 0 - provider.set_delay(delay) - - return provider - - def compare(self, other): - assert self.name == other.name - - for key, val in self.options.iteritems(): - if other.options.get(key, None) != val: - return False - - return True - - def hooks(self): - hooks = [] - parent = self._parent - if self.options["use_btrfs"]: - working_dir = parent.btrfs_working_dir_tmpl.format( - mirror_root=parent.mirror_root, - mirror_name=self.name - ) - service_dir = parent.btrfs_service_dir_tmpl.format( - mirror_root=parent.mirror_root, - mirror_name=self.name - ) - gc_dir = parent.btrfs_gc_dir_tmpl.format( - mirror_root=parent.mirror_root, - mirror_name=self.name - ) - hooks.append(BtrfsHook(service_dir, working_dir, gc_dir)) - - hooks.append(LogLimitHook()) - - if self.exec_pre_sync: - hooks.append( - CmdExecHook(self.exec_pre_sync, CmdExecHook.PRE_SYNC)) - - if self.exec_post_sync: - hooks.append( - CmdExecHook(self.exec_post_sync, CmdExecHook.POST_SYNC)) - - return hooks - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/mirror_provider.py b/tunasync/mirror_provider.py deleted file mode 100644 index ec093be..0000000 --- a/tunasync/mirror_provider.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import sh -import os -import shlex -from datetime import datetime - - -class MirrorProvider(object): - ''' - Mirror method class, can be `rsync', `debmirror', etc. - ''' - - def __init__(self, name, local_dir, log_dir, log_file="/dev/null", - interval=120, hooks=[]): - self.name = name - self.local_dir = local_dir - self.log_file = log_file - self.log_dir = log_dir - self.interval = interval - self.hooks = hooks - self.p = None - self.delay = 0 - - # deprecated - def ensure_log_dir(self): - log_dir = os.path.dirname(self.log_file) - if not os.path.exists(log_dir): - sh.mkdir("-p", log_dir) - - def get_log_file(self, ctx={}): - if 'log_file' in ctx: - log_file = ctx['log_file'] - else: - now = datetime.now().strftime("%Y-%m-%d_%H") - log_file = self.log_file.format(date=now) - ctx['log_file'] = log_file - return log_file - - def set_delay(self, sec): - ''' Set start delay ''' - self.delay = sec - - def run(self, ctx={}): - raise NotImplementedError("run method should be implemented") - - def terminate(self): - if self.p is not None: - self.p.process.terminate() - print("{} terminated".format(self.name)) - self.p = None - - def wait(self): - if self.p is not None: - self.p.wait() - self.p = None - - -class RsyncProvider(MirrorProvider): - - _default_options = ['-aHvh', '--no-o', '--no-g', '--stats', - '--exclude', '.~tmp~/', - '--delete', '--delete-after', '--delay-updates', - '--safe-links', '--timeout=120', '--contimeout=120'] - - def __init__(self, name, upstream_url, local_dir, log_dir, - useIPv6=True, password=None, exclude_file=None, - log_file="/dev/null", interval=120, env=None, hooks=[]): - super(RsyncProvider, self).__init__(name, local_dir, log_dir, log_file, - interval, hooks) - - self.upstream_url = upstream_url - self.useIPv6 = useIPv6 - self.exclude_file = exclude_file - self.password = password - self.env = env - - @property - def options(self): - - _options = [o for o in self._default_options] # copy - - if self.useIPv6: - _options.append("-6") - - if self.exclude_file: - _options.append("--exclude-from") - _options.append(self.exclude_file) - - return _options - - def run(self, ctx={}): - _args = self.options - _args.append(self.upstream_url) - - working_dir = ctx.get("current_dir", self.local_dir) - _args.append(working_dir) - - log_file = self.get_log_file(ctx) - new_env = os.environ.copy() - if self.password is not None: - new_env["RSYNC_PASSWORD"] = self.password - if self.env is not None and isinstance(self.env, dict): - for k, v in self.env.items(): - new_env[k] = v - - self.p = sh.rsync(*_args, _env=new_env, _out=log_file, - _err_to_out=True, _out_bufsize=1, _bg=True) - - -class TwoStageRsyncProvider(RsyncProvider): - - _stage1_options = ['-aHvh', '--no-o', '--no-g', - '--exclude', '.~tmp~/', - '--safe-links', '--timeout=120', '--contimeout=120'] - - _stage2_options = ['-aHvh', '--no-o', '--no-g', '--stats', - '--exclude', '.~tmp~/', - '--delete', '--delete-after', '--delay-updates', - '--safe-links', '--timeout=120', '--contimeout=120'] - - _stage1_profiles = { - "debian": [ - 'dists/', - ], - "debian-oldstyle": [ - 'Packages*', 'Sources*', 'Release*', - 'InRelease', 'i18n/*', 'ls-lR*', 'dep11/*', - ] - } - - def set_stage1_profile(self, profile): - if profile not in self._stage1_profiles: - raise Exception("Profile Undefined: %s, %s" % (profile, self.name)) - - self._stage1_excludes = self._stage1_profiles[profile] - - def options(self, stage): - _default_options = self._stage1_options \ - if stage == 1 else self._stage2_options - _options = [o for o in _default_options] # copy - - if stage == 1: - for _exc in self._stage1_excludes: - _options.append("--exclude") - _options.append(_exc) - - if self.useIPv6: - _options.append("-6") - - if self.exclude_file: - _options.append("--exclude-from") - _options.append(self.exclude_file) - - return _options - - def run(self, ctx={}): - working_dir = ctx.get("current_dir", self.local_dir) - log_file = self.get_log_file(ctx) - new_env = os.environ.copy() - if self.password is not None: - new_env["RSYNC_PASSWORD"] = self.password - if self.env is not None and isinstance(self.env, dict): - for k, v in self.env.items(): - new_env[k] = v - - with open(log_file, 'w', buffering=1) as f: - def log_output(line): - f.write(line) - - for stage in (1, 2): - - _args = self.options(stage) - _args.append(self.upstream_url) - _args.append(working_dir) - f.write("==== Stage {} Begins ====\n\n".format(stage)) - - self.p = sh.rsync( - *_args, _env=new_env, _out=log_output, - _err_to_out=True, _out_bufsize=1, _bg=False - ) - self.p.wait() - - -class ShellProvider(MirrorProvider): - - def __init__(self, name, command, upstream_url, local_dir, log_dir, - log_file="/dev/null", log_stdout=True, interval=120, env=None, - hooks=[]): - - super(ShellProvider, self).__init__(name, local_dir, log_dir, log_file, - interval, hooks) - self.upstream_url = str(upstream_url) - self.command = shlex.split(command) - self.log_stdout = log_stdout - self.env = env - - def run(self, ctx={}): - - log_file = self.get_log_file(ctx) - - new_env = os.environ.copy() - new_env["TUNASYNC_MIRROR_NAME"] = self.name - new_env["TUNASYNC_LOCAL_DIR"] = self.local_dir - new_env["TUNASYNC_WORKING_DIR"] = ctx.get("current_dir", self.local_dir) - new_env["TUNASYNC_UPSTREAM_URL"] = self.upstream_url - new_env["TUNASYNC_LOG_FILE"] = log_file - - if self.env is not None and isinstance(self.env, dict): - for k, v in self.env.items(): - new_env[k] = v - - _cmd = self.command[0] - _args = [] if len(self.command) == 1 else self.command[1:] - - cmd = sh.Command(_cmd) - - if self.log_stdout: - self.p = cmd(*_args, _env=new_env, _out=log_file, - _err_to_out=True, _out_bufsize=1, _bg=True) - else: - self.p = cmd(*_args, _env=new_env, _out='/dev/null', - _err='/dev/null', _out_bufsize=1, _bg=True) - - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/status_manager.py b/tunasync/status_manager.py deleted file mode 100644 index 46bad12..0000000 --- a/tunasync/status_manager.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import json -from datetime import datetime - - -class StatusManager(object): - - def __init__(self, parent, dbfile): - self.parent = parent - self.dbfile = dbfile - self.init_mirrors() - - def init_mirrors(self): - mirrors = {} - for name, cfg in self.parent.mirrors.iteritems(): - mirrors[name] = { - 'name': name, - 'last_update': '-', - 'status': 'unknown', - 'upstream': cfg.upstream or '-', - } - - try: - with open(self.dbfile) as f: - _mirrors = json.load(f) - for m in _mirrors: - name = m["name"] - mirrors[name]["last_update"] = m["last_update"] - mirrors[name]["status"] = m["status"] - except: - pass - - self.mirrors = mirrors - self.mirrors_ctx = {key: {} for key in self.mirrors} - - def get_info(self, name, key): - if key == "ctx": - return self.mirrors_ctx.get(name, {}) - _m = self.mirrors.get(name, {}) - return _m.get(key, None) - - def refresh_mirror(self, name): - cfg = self.parent.mirrors.get(name, None) - if cfg is None: - return - _m = self.mirrors.get(name, { - 'name': name, - 'last_update': '-', - 'status': '-', - }) - _m['upstream'] = cfg.upstream or '-' - self.mirrors[name] = dict(_m.items()) - self.commit_db() - - def update_status(self, name, status, ctx={}): - - _m = self.mirrors.get(name, { - 'name': name, - 'last_update': '-', - 'status': '-', - }) - - if status in ("syncing", "fail", "pre-syncing"): - update_time = _m["last_update"] - elif status == "success": - update_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - else: - print("Invalid status: {}, from {}".format(status, name)) - - _m['last_update'] = update_time - _m['status'] = status - self.mirrors[name] = dict(_m.items()) - self.mirrors_ctx[name] = ctx - - self.commit_db() - print("Updated status file, {}:{}".format(name, status)) - - def list_status(self, _format=False): - _mirrors = sorted( - [m for _, m in self.mirrors.items()], - key=lambda x: x['name'] - ) - if not _format: - return _mirrors - - name_len = max([len(_m['name']) for _m in _mirrors]) - update_len = max([len(_m['last_update']) for _m in _mirrors]) - status_len = max([len(_m['status']) for _m in _mirrors]) - heading = ' '.join([ - 'name'.ljust(name_len), - 'last update'.ljust(update_len), - 'status'.ljust(status_len) - ]) - line = ' '.join(['-'*name_len, '-'*update_len, '-'*status_len]) - tabular = '\n'.join( - [ - ' '.join( - (_m['name'].ljust(name_len), - _m['last_update'].ljust(update_len), - _m['status'].ljust(status_len)) - ) for _m in _mirrors - ] - ) - return '\n'.join((heading, line, tabular)) - - def get_status(self, name, _format=False): - if name not in self.mirrors: - return None - - mir = self.mirrors[name] - if not _format: - return mir - - tmpl = "{name} last_update: {last_update} status: {status}" - return tmpl.format(**mir) - - def commit_db(self): - with open(self.dbfile, 'wb') as f: - _mirrors = self.list_status() - json.dump(_mirrors, f, indent=2, separators=(',', ':')) - -# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tunasync/tunasync.py b/tunasync/tunasync.py deleted file mode 100644 index 5078cb3..0000000 --- a/tunasync/tunasync.py +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding:utf-8 -*- -import signal -import sys -import toml - -from multiprocessing import Process, Semaphore, Queue -from . import jobs -from .hook import JobHook -from .mirror_config import MirrorConfig -from .status_manager import StatusManager -from .clt_server import run_control_server - - -class TUNASync(object): - - _instance = None - _settings = None - _inited = False - - def __new__(cls, *args, **kwargs): - if not cls._instance: - cls._instance = super(TUNASync, cls).__new__(cls, *args, **kwargs) - - return cls._instance - - def read_config(self, config_file): - self._config_file = config_file - with open(self._config_file) as f: - self._settings = toml.loads(f.read()) - - self._inited = True - self._mirrors = {} - self._providers = {} - self.processes = {} - self.semaphore = Semaphore(self._settings["global"]["concurrent"]) - self.channel = Queue() - self._hooks = [] - - self.mirror_root = self._settings["global"]["mirror_root"] - - self.use_btrfs = self._settings["global"]["use_btrfs"] - self.btrfs_service_dir_tmpl = self._settings["btrfs"]["service_dir"] - self.btrfs_working_dir_tmpl = self._settings["btrfs"]["working_dir"] - self.btrfs_gc_dir_tmpl = self._settings["btrfs"]["gc_dir"] - - self.status_file = self._settings["global"]["status_file"] - self.status_manager = StatusManager(self, self.status_file) - - self.ctrl_addr = self._settings["global"]["ctrl_addr"] - self.ctrl_channel = Queue() - p = Process( - target=run_control_server, - args=(self.ctrl_addr, self.channel, self.ctrl_channel), - ) - p.start() - self.processes["CTRL_SERVER"] = (self.ctrl_channel, p) - - def add_hook(self, h): - assert isinstance(h, JobHook) - self._hooks.append(h) - - def hooks(self): - return self._hooks - - @property - def mirrors(self): - if self._mirrors: - return self._mirrors - - for mirror_opt in self._settings["mirrors"]: - name = mirror_opt["name"] - self._mirrors[name] = \ - MirrorConfig(self, mirror_opt) - - return self._mirrors - - @property - def providers(self): - if self._providers: - return self._providers - - for name, mirror in self.mirrors.iteritems(): - hooks = mirror.hooks() + self.hooks() - provider = mirror.to_provider(hooks, no_delay=mirror.no_delay) - self._providers[name] = provider - - return self._providers - - def run_jobs(self): - for name in self.providers: - self.run_provider(name) - - def sig_handler(*args): - print("terminate subprocesses") - for _, np in self.processes.iteritems(): - _, p = np - p.terminate() - print("Good Bye") - sys.exit(0) - - signal.signal(signal.SIGINT, sig_handler) - signal.signal(signal.SIGTERM, sig_handler) - signal.signal(signal.SIGUSR1, self.reload_mirrors) - signal.signal(signal.SIGUSR2, self.reload_mirrors_force) - - self.run_forever() - - def run_provider(self, name): - if name not in self.providers: - print("{} doesnot exist".format(name)) - return - - provider = self.providers[name] - child_queue = Queue() - p = Process( - target=jobs.run_job, - args=(self.semaphore, child_queue, self.channel, provider, ), - kwargs={ - 'max_retry': self._settings['global']['max_retry']} - ) - p.start() - provider.set_delay(0) # clear delay after first start - self.processes[name] = (child_queue, p) - - def reload_mirrors(self, signum, frame): - try: - return self._reload_mirrors(signum, frame, force=False) - except Exception as e: - print(e) - - def reload_mirrors_force(self, signum, frame): - try: - return self._reload_mirrors(signum, frame, force=True) - except Exception as e: - print(e) - - def _reload_mirrors(self, signum, frame, force=False): - print("reload mirror configs, force restart: {}".format(force)) - - with open(self._config_file) as f: - self._settings = toml.loads(f.read()) - - for mirror_opt in self._settings["mirrors"]: - name = mirror_opt["name"] - newMirCfg = MirrorConfig(self, mirror_opt) - - if name in self._mirrors: - if newMirCfg.compare(self._mirrors[name]): - continue - - self._mirrors[name] = newMirCfg - - hooks = newMirCfg.hooks() + self.hooks() - newProvider = newMirCfg.to_provider(hooks, no_delay=True) - self._providers[name] = newProvider - - if name in self.processes: - q, p = self.processes[name] - - if force: - p.terminate() - print("Terminated Job: {}".format(name)) - self.run_provider(name) - else: - q.put("terminate") - print("New configuration queued to {}".format(name)) - else: - print("New mirror: {}".format(name)) - self.run_provider(name) - - self.status_manager.refresh_mirror(name) - - def run_forever(self): - while 1: - try: - msg_hdr, msg_body = self.channel.get() - except IOError: - continue - - if msg_hdr == "UPDATE": - mirror_name, status, ctx = msg_body - try: - self.status_manager.update_status( - mirror_name, status, dict(ctx.items())) - except Exception as e: - print(e) - - elif msg_hdr == "CONFIG_ACK": - mirror_name, status = msg_body - if status == "QUIT": - print("New configuration applied to {}".format(mirror_name)) - self.run_provider(mirror_name) - - elif msg_hdr == "CMD": - cmd, mirror_name, kwargs = msg_body - if (mirror_name not in self.mirrors) and (mirror_name != "__ALL__"): - self.ctrl_channel.put("Invalid target") - continue - res = self.handle_cmd(cmd, mirror_name, kwargs) - self.ctrl_channel.put(res) - - def handle_cmd(self, cmd, mirror_name, kwargs): - if cmd == "restart": - if mirror_name not in self.providers: - res = "Invalid job: {}".format(mirror_name) - return res - - if mirror_name in self.processes: - _, p = self.processes[mirror_name] - p.terminate() - self.providers[mirror_name].set_delay(0) - self.run_provider(mirror_name) - res = "Restarted Job: {}".format(mirror_name) - - elif cmd == "stop": - if mirror_name not in self.processes: - res = "{} not running".format(mirror_name) - return res - - _, p = self.processes.pop(mirror_name) - p.terminate() - res = "Stopped Job: {}".format(mirror_name) - - elif cmd == "start": - if mirror_name in self.processes: - res = "{} already running".format(mirror_name) - return res - - self.run_provider(mirror_name) - res = "Started Job: {}".format(mirror_name) - - elif cmd == "status": - if mirror_name == "__ALL__": - res = self.status_manager.list_status(_format=True) - else: - res = self.status_manager.get_status(mirror_name, _format=True) - - elif cmd == "log": - job_ctx = self.status_manager.get_info(mirror_name, "ctx") - n = kwargs.get("n", 0) - if n == 0: - res = job_ctx.get( - "log_link", - job_ctx.get("log_file", "/dev/null"), - ) - else: - import os - log_file = job_ctx.get("log_file", None) - if log_file is None: - return "/dev/null" - - log_dir = os.path.dirname(log_file) - lfiles = [ - os.path.join(log_dir, lfile) - for lfile in os.listdir(log_dir) - if lfile.startswith(mirror_name) and lfile != "latest" - ] - - if len(lfiles) <= n: - res = "Only {} log files available".format(len(lfiles)) - return res - - lfiles_set = set(lfiles) - # sort to get the newest 10 files - lfiles_ts = sorted( - [(os.path.getmtime(lfile), lfile) for lfile in lfiles_set], - key=lambda x: x[0], - reverse=True, - ) - return lfiles_ts[n][1] - - else: - res = "Invalid command" - - return res - - -# vim: ts=4 sw=4 sts=4 expandtab From 5ef50090e92ee11d6cbf97c50414b29950cc9154 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 22:09:55 +0800 Subject: [PATCH 63/66] docs: update README --- README.md | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 485cba9..8fde1d2 100644 --- a/README.md +++ b/README.md @@ -12,35 +12,40 @@ tunasync - Manager: Centural instance on status and job management - Worker: Runs mirror jobs - -+----------+ +---+ worker configs +---+ +----------+ +----------+ -| Status | | |+-----------------> | w +--->| mirror +---->| mirror | -| Manager | | | | o | | config | | provider | -+----------+ | W | start/stop job | r | +----------+ +----+-----+ - | E |+-----------------> | k | | -+----------+ | B | | e | +------------+ | -| Job | | | update status | r |<------+ mirror job |<----+ -|Controller| | | <-----------------+| | +------------+ -+----------+ +---+ +---+ ++------------+ +---+ +---+ +| Client API | | | Job Status | | +----------+ +----------+ ++------------+ | +----------------->| |--->| mirror +---->| mirror | ++------------+ | | | w | | config | | provider | +| Worker API | | H | | o | +----------+ +----+-----+ ++------------+ | T | Job Control | r | | ++------------+ | T +----------------->| k | +------------+ | +| Job/Status | | P | Start/Stop/... | e | | mirror job |<----+ +| Management | | S | | r | +------^-----+ ++------------+ | | Update Status | | +---------+---------+ ++------------+ | <------------------+ | | Scheduler | +| BoltDB | | | | | +-------------------+ ++------------+ +---+ +---+ # Job Run Process + +PreSyncing Syncing Success +-----------+ +-----------+ +-------------+ +--------------+ -| pre-job +--+->| job run +--->| post-job +-+-->| post-success | +| pre-job +--+->| job run +--->| post-exec +-+-->| post-success | +-----------+ ^ +-----------+ +-------------+ | +--------------+ | | - | +-----------------+ | + | +-----------------+ | Failed +------+ post-fail |<---------+ +-----------------+ ``` ## TODO -- [ ] split to `tunasync-manager` and `tunasync-worker` instances - - [ ] use HTTP as communication protocol - - [ ] implement manager as status server first, and use python worker - - [ ] implement go worker +- [x] split to `tunasync-manager` and `tunasync-worker` instances + - [x] use HTTP as communication protocol + - [x] implement manager as status server first, and use python worker + - [x] implement go worker - Web frontend for `tunasync-manager` - [ ] start/stop/restart job - [ ] enable/disable mirror @@ -48,6 +53,7 @@ tunasync - [ ] config file structure - [ ] support multi-file configuration (`/etc/tunasync.d/mirror-enabled/*.conf`) + ## Generate Self-Signed Certificate Fisrt, create root CA From 839941788079a3a790ce74c367710bccecbc3412 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 22:11:59 +0800 Subject: [PATCH 64/66] feature(worker): fix exec_post_hook_test, manager channel was blocked in the previous version --- worker/exec_post_test.go | 11 ++++++----- worker/job_test.go | 12 ++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/worker/exec_post_test.go b/worker/exec_post_test.go index e2f7efc..203c607 100644 --- a/worker/exec_post_test.go +++ b/worker/exec_post_test.go @@ -82,7 +82,6 @@ echo $TUNASYNC_WORKING_DIR echo $TUNASYNC_MIRROR_NAME echo $TUNASYNC_UPSTREAM_URL echo $TUNASYNC_LOG_FILE -sleep 5 exit 1 ` @@ -93,10 +92,12 @@ exit 1 job.ctrlChan <- jobStart msg := <-managerChan So(msg.status, ShouldEqual, PreSyncing) - msg = <-managerChan - So(msg.status, ShouldEqual, Syncing) - msg = <-managerChan - So(msg.status, ShouldEqual, Failed) + for i := 0; i < maxRetry; i++ { + msg = <-managerChan + So(msg.status, ShouldEqual, Syncing) + msg = <-managerChan + So(msg.status, ShouldEqual, Failed) + } time.Sleep(200 * time.Millisecond) job.ctrlChan <- jobDisable diff --git a/worker/job_test.go b/worker/job_test.go index 1edc14b..7ffed72 100644 --- a/worker/job_test.go +++ b/worker/job_test.go @@ -49,7 +49,7 @@ func TestMirrorJob(t *testing.T) { echo $TUNASYNC_UPSTREAM_URL echo $TUNASYNC_LOG_FILE ` - exceptedOutput := fmt.Sprintf( + expectedOutput := fmt.Sprintf( "%s\n%s\n%s\n%s\n", provider.WorkingDir(), provider.Name(), @@ -86,7 +86,7 @@ func TestMirrorJob(t *testing.T) { So(msg.status, ShouldEqual, Success) loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) - So(string(loggedContent), ShouldEqual, exceptedOutput) + So(string(loggedContent), ShouldEqual, expectedOutput) job.ctrlChan <- jobStart } select { @@ -140,10 +140,10 @@ echo $TUNASYNC_WORKING_DIR msg = <-managerChan So(msg.status, ShouldEqual, Failed) - exceptedOutput := fmt.Sprintf("%s\n", provider.WorkingDir()) + expectedOutput := fmt.Sprintf("%s\n", provider.WorkingDir()) loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) - So(string(loggedContent), ShouldEqual, exceptedOutput) + So(string(loggedContent), ShouldEqual, expectedOutput) job.ctrlChan <- jobDisable <-job.disabled }) @@ -159,14 +159,14 @@ echo $TUNASYNC_WORKING_DIR msg = <-managerChan So(msg.status, ShouldEqual, Success) - exceptedOutput := fmt.Sprintf( + expectedOutput := fmt.Sprintf( "%s\n%s\n", provider.WorkingDir(), provider.WorkingDir(), ) loggedContent, err := ioutil.ReadFile(provider.LogFile()) So(err, ShouldBeNil) - So(string(loggedContent), ShouldEqual, exceptedOutput) + So(string(loggedContent), ShouldEqual, expectedOutput) job.ctrlChan <- jobDisable <-job.disabled }) From 56459f2ce03e70b28688f2aefac035cde045f837 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Fri, 29 Apr 2016 22:38:47 +0800 Subject: [PATCH 65/66] feature(worker): implemented mirror role (master/slave) option --- tests/worker.conf | 1 + worker/config.go | 1 + worker/provider.go | 6 ++++++ worker/worker.go | 15 ++++++++++++++- 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/worker.conf b/tests/worker.conf index 2a27567..42550a0 100644 --- a/tests/worker.conf +++ b/tests/worker.conf @@ -29,6 +29,7 @@ command = "/tmp/tunasync/bin/myrsync2.sh" upstream = "https://aosp.google.com/" interval = 2 mirror_dir = "/tmp/tunasync/git/AOSP" +role = "slave" [mirrors.env] REPO = "/usr/local/bin/aosp-repo" diff --git a/worker/config.go b/worker/config.go index 96f4d28..0a37210 100644 --- a/worker/config.go +++ b/worker/config.go @@ -75,6 +75,7 @@ type mirrorConfig struct { MirrorDir string `toml:"mirror_dir"` LogDir string `toml:"log_dir"` Env map[string]string `toml:"env"` + Role string `toml:"role"` ExecOnSuccess string `toml:"exec_on_success"` ExecOnFailure string `toml:"exec_on_failure"` diff --git a/worker/provider.go b/worker/provider.go index 6686290..3a44b04 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -44,6 +44,7 @@ type mirrorProvider interface { WorkingDir() string LogDir() string LogFile() string + IsMaster() bool // enter context EnterContext() *Context @@ -59,6 +60,7 @@ type baseProvider struct { ctx *Context name string interval time.Duration + isMaster bool cmd *cmdJob isRunning atomic.Value @@ -92,6 +94,10 @@ func (p *baseProvider) Interval() time.Duration { return p.interval } +func (p *baseProvider) IsMaster() bool { + return p.isMaster +} + func (p *baseProvider) WorkingDir() string { if v, ok := p.ctx.Get(_WorkingDirKey); ok { if s, ok := v.(string); ok { diff --git a/worker/worker.go b/worker/worker.go index 64d0bbf..19bc067 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -90,6 +90,16 @@ func (w *Worker) initProviders() { } logDir = formatLogDir(logDir, mirror) + // IsMaster + isMaster := true + if mirror.Role == "slave" { + isMaster = false + } else { + if mirror.Role != "" && mirror.Role != "master" { + logger.Warningf("Invalid role configuration for %s", mirror.Name) + } + } + var provider mirrorProvider switch mirror.Provider { @@ -105,6 +115,7 @@ func (w *Worker) initProviders() { env: mirror.Env, } p, err := newCmdProvider(pc) + p.isMaster = isMaster if err != nil { panic(err) } @@ -123,6 +134,7 @@ func (w *Worker) initProviders() { interval: time.Duration(mirror.Interval) * time.Minute, } p, err := newRsyncProvider(rc) + p.isMaster = isMaster if err != nil { panic(err) } @@ -142,6 +154,7 @@ func (w *Worker) initProviders() { interval: time.Duration(mirror.Interval) * time.Minute, } p, err := newTwoStageRsyncProvider(rc) + p.isMaster = isMaster if err != nil { panic(err) } @@ -395,7 +408,7 @@ func (w *Worker) updateStatus(jobMsg jobMessage) { smsg := MirrorStatus{ Name: jobMsg.name, Worker: w.cfg.Global.Name, - IsMaster: true, + IsMaster: p.IsMaster(), Status: jobMsg.status, Upstream: p.Upstream(), Size: "unknown", From 2ea22ec1ae825a588baa5812443a6a83bcd0a93a Mon Sep 17 00:00:00 2001 From: walkerning Date: Sat, 30 Apr 2016 00:38:18 +0800 Subject: [PATCH 66/66] feature(cmd): add default config files for tunasynctl --- cmd/tunasync/tunasync.go | 8 ++-- cmd/tunasynctl/tunasynctl.go | 81 +++++++++++++++++++++++++++--------- 2 files changed, 65 insertions(+), 24 deletions(-) diff --git a/cmd/tunasync/tunasync.go b/cmd/tunasync/tunasync.go index dfe45d8..7000126 100644 --- a/cmd/tunasync/tunasync.go +++ b/cmd/tunasync/tunasync.go @@ -19,7 +19,7 @@ func startManager(c *cli.Context) { cfg, err := manager.LoadConfig(c.String("config"), c) if err != nil { - logger.Error("Error loading config: %s", err.Error()) + logger.Errorf("Error loading config: %s", err.Error()) os.Exit(1) } if !cfg.Debug { @@ -28,7 +28,7 @@ func startManager(c *cli.Context) { m := manager.GetTUNASyncManager(cfg) if m == nil { - logger.Error("Error intializing TUNA sync worker.") + logger.Errorf("Error intializing TUNA sync worker.") os.Exit(1) } @@ -44,13 +44,13 @@ func startWorker(c *cli.Context) { cfg, err := worker.LoadConfig(c.String("config")) if err != nil { - logger.Error("Error loading config: %s", err.Error()) + logger.Errorf("Error loading config: %s", err.Error()) os.Exit(1) } w := worker.GetTUNASyncWorker(cfg) if w == nil { - logger.Error("Error intializing TUNA sync worker.") + logger.Errorf("Error intializing TUNA sync worker.") os.Exit(1) } diff --git a/cmd/tunasynctl/tunasynctl.go b/cmd/tunasynctl/tunasynctl.go index f4ddc54..863a30d 100644 --- a/cmd/tunasynctl/tunasynctl.go +++ b/cmd/tunasynctl/tunasynctl.go @@ -8,6 +8,7 @@ import ( "os" "strings" + "github.com/BurntSushi/toml" "github.com/codegangsta/cli" "gopkg.in/op/go-logging.v1" @@ -18,6 +19,9 @@ const ( listJobsPath = "/jobs" listWorkersPath = "/workers" cmdPath = "/cmd" + + systemCfgFile = "/etc/tunasync/ctl.conf" + userCfgFile = "$HOME/.config/tunasync/ctl.conf" ) var logger = logging.MustGetLogger("tunasynctl-cmd") @@ -35,29 +39,65 @@ func initializeWrapper(handler func(*cli.Context)) func(*cli.Context) { } } +type config struct { + ManagerAddr string `toml:"manager_addr"` + ManagerPort int `toml:"manager_port"` + CACert string `toml:"ca_cert"` +} + +func loadConfig(cfgFile string, c *cli.Context) (*config, error) { + cfg := new(config) + cfg.ManagerAddr = "localhost" + cfg.ManagerPort = 14242 + + if cfgFile != "" { + if _, err := toml.DecodeFile(cfgFile, cfg); err != nil { + logger.Errorf(err.Error()) + return nil, err + } + } + + if c.String("manager") != "" { + cfg.ManagerAddr = c.String("manager") + } + if c.Int("port") > 0 { + cfg.ManagerPort = c.Int("port") + } + + if c.String("ca-cert") != "" { + cfg.CACert = c.String("ca-cert") + } + return cfg, nil +} + func initialize(c *cli.Context) error { // init logger tunasync.InitLogger(c.Bool("verbose"), c.Bool("verbose"), false) + var cfgFile string - // parse manager server address - baseURL = c.String("manager") - if baseURL == "" { - baseURL = "localhost" + // choose config file and load config + if c.String("config") != "" { + cfgFile = c.String("config") + } else if _, err := os.Stat(os.ExpandEnv(userCfgFile)); err == nil { + cfgFile = os.ExpandEnv(userCfgFile) + } else if _, err := os.Stat(systemCfgFile); err == nil { + cfgFile = systemCfgFile } - managerPort := c.String("port") - if managerPort != "" { - baseURL += ":" + managerPort - } - if c.Bool("no-ssl") { - baseURL = "http://" + baseURL - } else { - baseURL = "https://" + baseURL + cfg, err := loadConfig(cfgFile, c) + + if err != nil { + logger.Errorf("Load configuration for tunasynctl error: %s", err.Error()) + return err } + + // parse base url of the manager server + baseURL = fmt.Sprintf("https://%s:%d", + cfg.ManagerAddr, cfg.ManagerPort) + logger.Infof("Use manager address: %s", baseURL) // create HTTP client - var err error - client, err = tunasync.CreateHTTPClient(c.String("ca-cert")) + client, err = tunasync.CreateHTTPClient(cfg.CACert) if err != nil { err = fmt.Errorf("Error initializing HTTP client: %s", err.Error()) logger.Error(err.Error()) @@ -169,6 +209,11 @@ func main() { app.Version = "0.1" commonFlags := []cli.Flag{ + cli.StringFlag{ + Name: "config, c", + Usage: "Read configuration from `FILE` rather than" + + " ~/.config/tunasync/ctl.conf and /etc/tunasync/ctl.conf", + }, cli.StringFlag{ Name: "manager, m", Usage: "The manager server address", @@ -178,14 +223,10 @@ func main() { Usage: "The manager server port", }, cli.StringFlag{ - Name: "ca-cert, c", - Usage: "Trust CA cert `CERT`", + Name: "ca-cert", + Usage: "Trust root CA cert file `CERT`", }, - cli.BoolFlag{ - Name: "no-ssl", - Usage: "Use http rather than https", - }, cli.BoolFlag{ Name: "verbose, v", Usage: "Enable verbosely logging",