diff --git a/cmd/tunasynctl/tunasynctl.go b/cmd/tunasynctl/tunasynctl.go index ea12dcf..f335971 100644 --- a/cmd/tunasynctl/tunasynctl.go +++ b/cmd/tunasynctl/tunasynctl.go @@ -140,8 +140,7 @@ func listWorkers(c *cli.Context) error { } func listJobs(c *cli.Context) error { - // FIXME: there should be an API on manager server side that return MirrorStatus list to tunasynctl - var jobs []tunasync.MirrorStatus + var jobs []tunasync.WebMirrorStatus if c.Bool("all") { _, err := tunasync.GetJSON(baseURL+listJobsPath, &jobs, client) if err != nil { @@ -158,10 +157,10 @@ func listJobs(c *cli.Context) error { fmt.Sprintf("Usage Error: jobs command need at"+ " least one arguments or \"--all\" flag."), 1) } - ans := make(chan []tunasync.MirrorStatus, len(args)) + ans := make(chan []tunasync.WebMirrorStatus, len(args)) for _, workerID := range args { go func(workerID string) { - var workerJobs []tunasync.MirrorStatus + var workerJobs []tunasync.WebMirrorStatus _, err := tunasync.GetJSON(fmt.Sprintf("%s/workers/%s/jobs", baseURL, workerID), &workerJobs, client) if err != nil { diff --git a/internal/msg.go b/internal/msg.go index 15791b9..c0efb7a 100644 --- a/internal/msg.go +++ b/internal/msg.go @@ -13,6 +13,7 @@ type MirrorStatus struct { IsMaster bool `json:"is_master"` Status SyncStatus `json:"status"` LastUpdate time.Time `json:"last_update"` + LastEnded time.Time `json:"last_ended"` Upstream string `json:"upstream"` Size string `json:"size"` ErrorMsg string `json:"error_msg"` diff --git a/manager/status.go b/internal/status_web.go similarity index 76% rename from manager/status.go rename to internal/status_web.go index 31bd1d5..9329c96 100644 --- a/manager/status.go +++ b/internal/status_web.go @@ -1,11 +1,9 @@ -package manager +package internal import ( "encoding/json" "strconv" "time" - - . "github.com/tuna/tunasync/internal" ) type textTime struct { @@ -38,24 +36,28 @@ func (t *stampTime) UnmarshalJSON(b []byte) error { return err } -// webMirrorStatus is the mirror status to be shown in the web page -type webMirrorStatus struct { +// WebMirrorStatus is the mirror status to be shown in the web page +type WebMirrorStatus struct { Name string `json:"name"` IsMaster bool `json:"is_master"` Status SyncStatus `json:"status"` LastUpdate textTime `json:"last_update"` LastUpdateTs stampTime `json:"last_update_ts"` + LastEnded textTime `json:"last_ended"` + LastEndedTs stampTime `json:"last_ended_ts"` Upstream string `json:"upstream"` Size string `json:"size"` // approximate size } -func convertMirrorStatus(m MirrorStatus) webMirrorStatus { - return webMirrorStatus{ +func BuildWebMirrorStatus(m MirrorStatus) WebMirrorStatus { + return WebMirrorStatus{ Name: m.Name, IsMaster: m.IsMaster, Status: m.Status, LastUpdate: textTime{m.LastUpdate}, LastUpdateTs: stampTime{m.LastUpdate}, + LastEnded: textTime{m.LastEnded}, + LastEndedTs: stampTime{m.LastEnded}, Upstream: m.Upstream, Size: m.Size, } diff --git a/internal/status_web_test.go b/internal/status_web_test.go new file mode 100644 index 0000000..97453ed --- /dev/null +++ b/internal/status_web_test.go @@ -0,0 +1,76 @@ +package internal + +import ( + "encoding/json" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestStatus(t *testing.T) { + Convey("status json ser-de should work", t, func() { + tz := "Asia/Tokyo" + loc, err := time.LoadLocation(tz) + So(err, ShouldBeNil) + t := time.Date(2016, time.April, 16, 23, 8, 10, 0, loc) + m := WebMirrorStatus{ + Name: "tunalinux", + Status: Success, + LastUpdate: textTime{t}, + LastUpdateTs: stampTime{t}, + LastEnded: textTime{t}, + LastEndedTs: stampTime{t}, + Size: "5GB", + Upstream: "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/", + } + + b, err := json.Marshal(m) + So(err, ShouldBeNil) + //fmt.Println(string(b)) + var m2 WebMirrorStatus + err = json.Unmarshal(b, &m2) + So(err, ShouldBeNil) + // fmt.Printf("%#v", m2) + So(m2.Name, ShouldEqual, m.Name) + So(m2.Status, ShouldEqual, m.Status) + So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix()) + So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix()) + So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) + So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) + So(m2.LastEnded.Unix(), ShouldEqual, m.LastEnded.Unix()) + So(m2.LastEndedTs.Unix(), ShouldEqual, m.LastEnded.Unix()) + So(m2.LastEnded.UnixNano(), ShouldEqual, m.LastEnded.UnixNano()) + So(m2.LastEndedTs.UnixNano(), ShouldEqual, m.LastEnded.UnixNano()) + So(m2.Size, ShouldEqual, m.Size) + So(m2.Upstream, ShouldEqual, m.Upstream) + }) + Convey("BuildWebMirrorStatus should work", t, func() { + m := MirrorStatus{ + Name: "arch-sync3", + Worker: "testWorker", + IsMaster: true, + Status: Failed, + LastUpdate: time.Now().Add(-time.Minute * 30), + LastEnded: time.Now(), + Upstream: "mirrors.tuna.tsinghua.edu.cn", + Size: "4GB", + } + + var m2 WebMirrorStatus + m2 = BuildWebMirrorStatus(m) + // fmt.Printf("%#v", m2) + So(m2.Name, ShouldEqual, m.Name) + So(m2.Status, ShouldEqual, m.Status) + So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix()) + So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix()) + So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) + So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) + So(m2.LastEnded.Unix(), ShouldEqual, m.LastEnded.Unix()) + So(m2.LastEndedTs.Unix(), ShouldEqual, m.LastEnded.Unix()) + So(m2.LastEnded.UnixNano(), ShouldEqual, m.LastEnded.UnixNano()) + So(m2.LastEndedTs.UnixNano(), ShouldEqual, m.LastEnded.UnixNano()) + So(m2.Size, ShouldEqual, m.Size) + So(m2.Upstream, ShouldEqual, m.Upstream) + }) +} diff --git a/manager/db_test.go b/manager/db_test.go index 1d13eec..5e6e79d 100644 --- a/manager/db_test.go +++ b/manager/db_test.go @@ -65,6 +65,7 @@ func TestBoltAdapter(t *testing.T) { IsMaster: true, Status: Success, LastUpdate: time.Now(), + LastEnded: time.Now(), Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "3GB", }, @@ -73,7 +74,8 @@ func TestBoltAdapter(t *testing.T) { Worker: testWorkerIDs[1], IsMaster: true, Status: Disabled, - LastUpdate: time.Now(), + LastUpdate: time.Now().Add(-time.Hour), + LastEnded: time.Now(), Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "4GB", }, @@ -82,7 +84,8 @@ func TestBoltAdapter(t *testing.T) { Worker: testWorkerIDs[1], IsMaster: true, Status: Success, - LastUpdate: time.Now(), + LastUpdate: time.Now().Add(-time.Second), + LastEnded: time.Now(), Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "4GB", }, diff --git a/manager/server.go b/manager/server.go index 05c56bf..2563582 100644 --- a/manager/server.go +++ b/manager/server.go @@ -135,11 +135,11 @@ func (s *Manager) listAllJobs(c *gin.Context) { s.returnErrJSON(c, http.StatusInternalServerError, err) return } - webMirStatusList := []webMirrorStatus{} + webMirStatusList := []WebMirrorStatus{} for _, m := range mirrorStatusList { webMirStatusList = append( webMirStatusList, - convertMirrorStatus(m), + BuildWebMirrorStatus(m), ) } c.JSON(http.StatusOK, webMirStatusList) @@ -242,6 +242,11 @@ func (s *Manager) updateJobOfWorker(c *gin.Context) { } else { status.LastUpdate = curStatus.LastUpdate } + if status.Status == Success || status.Status == Failed { + status.LastEnded = time.Now() + } else { + status.LastEnded = curStatus.LastEnded + } // Only message with meaningful size updates the mirror size if len(curStatus.Size) > 0 && curStatus.Size != "unknown" { diff --git a/manager/server_test.go b/manager/server_test.go index c1122b0..27f3cd4 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -121,11 +121,12 @@ func TestHTTPServer(t *testing.T) { So(m.Size, ShouldEqual, status.Size) So(m.IsMaster, ShouldEqual, status.IsMaster) So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second) + So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second) }) Convey("list all job status of all workers", func(ctx C) { - var ms []webMirrorStatus + var ms []WebMirrorStatus resp, err := GetJSON(baseURL+"/jobs", &ms, nil) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) @@ -137,6 +138,7 @@ func TestHTTPServer(t *testing.T) { So(m.Size, ShouldEqual, status.Size) So(m.IsMaster, ShouldEqual, status.IsMaster) So(time.Now().Sub(m.LastUpdate.Time), ShouldBeLessThan, 1*time.Second) + So(time.Now().Sub(m.LastEnded.Time), ShouldBeLessThan, 1*time.Second) }) @@ -166,6 +168,7 @@ func TestHTTPServer(t *testing.T) { So(m.Size, ShouldEqual, "5GB") So(m.IsMaster, ShouldEqual, status.IsMaster) So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second) + So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second) }) }) @@ -180,6 +183,32 @@ func TestHTTPServer(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError) }) + + // what if status changed to failed + status.Status = Failed + time.Sleep(3 * time.Second) + resp, err = PostJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", baseURL, status.Worker, status.Name), status, nil) + defer resp.Body.Close() + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + + Convey("What if syncing job failed", func(ctx C) { + var ms []MirrorStatus + resp, err := GetJSON(baseURL+"/workers/test_worker1/jobs", &ms, nil) + + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + // err = json.NewDecoder(resp.Body).Decode(&mirrorStatusList) + m := ms[0] + So(m.Name, ShouldEqual, status.Name) + So(m.Worker, ShouldEqual, status.Worker) + So(m.Status, ShouldEqual, status.Status) + So(m.Upstream, ShouldEqual, status.Upstream) + So(m.Size, ShouldEqual, status.Size) + So(m.IsMaster, ShouldEqual, status.IsMaster) + So(time.Now().Sub(m.LastUpdate), ShouldBeGreaterThan, 3*time.Second) + So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second) + }) }) Convey("update mirror status of an inexisted worker", func(ctx C) { @@ -190,6 +219,7 @@ func TestHTTPServer(t *testing.T) { IsMaster: true, Status: Success, LastUpdate: time.Now(), + LastEnded: time.Now(), Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "4GB", } diff --git a/manager/status_test.go b/manager/status_test.go deleted file mode 100644 index 9cd046a..0000000 --- a/manager/status_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package manager - -import ( - "encoding/json" - "testing" - "time" - - tunasync "github.com/tuna/tunasync/internal" - - . "github.com/smartystreets/goconvey/convey" -) - -func TestStatus(t *testing.T) { - Convey("status json ser-de should work", t, func() { - tz := "Asia/Tokyo" - loc, err := time.LoadLocation(tz) - So(err, ShouldBeNil) - t := time.Date(2016, time.April, 16, 23, 8, 10, 0, loc) - m := webMirrorStatus{ - Name: "tunalinux", - Status: tunasync.Success, - LastUpdate: textTime{t}, - LastUpdateTs: stampTime{t}, - Size: "5GB", - Upstream: "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/", - } - - b, err := json.Marshal(m) - So(err, ShouldBeNil) - //fmt.Println(string(b)) - var m2 webMirrorStatus - err = json.Unmarshal(b, &m2) - So(err, ShouldBeNil) - // fmt.Printf("%#v", m2) - So(m2.Name, ShouldEqual, m.Name) - So(m2.Status, ShouldEqual, m.Status) - So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix()) - So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix()) - So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) - So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) - So(m2.Size, ShouldEqual, m.Size) - So(m2.Upstream, ShouldEqual, m.Upstream) - }) -}