From 4fe7d03e54eb09c82d6856aa855bb7097ff4765f Mon Sep 17 00:00:00 2001 From: Yuxiang Zhang Date: Tue, 29 May 2018 18:48:33 +0800 Subject: [PATCH 1/3] Move the WebMirrorStatus to internal package. Fix the list command of tunasynctl --- cmd/tunasynctl/tunasynctl.go | 7 +++---- manager/status.go => internal/status_web.go | 12 +++++------- .../status_test.go => internal/status_web_test.go | 10 ++++------ manager/server.go | 4 ++-- 4 files changed, 14 insertions(+), 19 deletions(-) rename manager/status.go => internal/status_web.go (83%) rename manager/status_test.go => internal/status_web_test.go (88%) diff --git a/cmd/tunasynctl/tunasynctl.go b/cmd/tunasynctl/tunasynctl.go index ea12dcf..f335971 100644 --- a/cmd/tunasynctl/tunasynctl.go +++ b/cmd/tunasynctl/tunasynctl.go @@ -140,8 +140,7 @@ func listWorkers(c *cli.Context) error { } func listJobs(c *cli.Context) error { - // FIXME: there should be an API on manager server side that return MirrorStatus list to tunasynctl - var jobs []tunasync.MirrorStatus + var jobs []tunasync.WebMirrorStatus if c.Bool("all") { _, err := tunasync.GetJSON(baseURL+listJobsPath, &jobs, client) if err != nil { @@ -158,10 +157,10 @@ func listJobs(c *cli.Context) error { fmt.Sprintf("Usage Error: jobs command need at"+ " least one arguments or \"--all\" flag."), 1) } - ans := make(chan []tunasync.MirrorStatus, len(args)) + ans := make(chan []tunasync.WebMirrorStatus, len(args)) for _, workerID := range args { go func(workerID string) { - var workerJobs []tunasync.MirrorStatus + var workerJobs []tunasync.WebMirrorStatus _, err := tunasync.GetJSON(fmt.Sprintf("%s/workers/%s/jobs", baseURL, workerID), &workerJobs, client) if err != nil { diff --git a/manager/status.go b/internal/status_web.go similarity index 83% rename from manager/status.go rename to internal/status_web.go index 31bd1d5..33e15ca 100644 --- a/manager/status.go +++ b/internal/status_web.go @@ -1,11 +1,9 @@ -package manager +package internal import ( "encoding/json" "strconv" "time" - - . "github.com/tuna/tunasync/internal" ) type textTime struct { @@ -38,8 +36,8 @@ func (t *stampTime) UnmarshalJSON(b []byte) error { return err } -// webMirrorStatus is the mirror status to be shown in the web page -type webMirrorStatus struct { +// WebMirrorStatus is the mirror status to be shown in the web page +type WebMirrorStatus struct { Name string `json:"name"` IsMaster bool `json:"is_master"` Status SyncStatus `json:"status"` @@ -49,8 +47,8 @@ type webMirrorStatus struct { Size string `json:"size"` // approximate size } -func convertMirrorStatus(m MirrorStatus) webMirrorStatus { - return webMirrorStatus{ +func BuildWebMirrorStatus(m MirrorStatus) WebMirrorStatus { + return WebMirrorStatus{ Name: m.Name, IsMaster: m.IsMaster, Status: m.Status, diff --git a/manager/status_test.go b/internal/status_web_test.go similarity index 88% rename from manager/status_test.go rename to internal/status_web_test.go index 9cd046a..ce31630 100644 --- a/manager/status_test.go +++ b/internal/status_web_test.go @@ -1,12 +1,10 @@ -package manager +package internal import ( "encoding/json" "testing" "time" - tunasync "github.com/tuna/tunasync/internal" - . "github.com/smartystreets/goconvey/convey" ) @@ -16,9 +14,9 @@ func TestStatus(t *testing.T) { loc, err := time.LoadLocation(tz) So(err, ShouldBeNil) t := time.Date(2016, time.April, 16, 23, 8, 10, 0, loc) - m := webMirrorStatus{ + m := WebMirrorStatus{ Name: "tunalinux", - Status: tunasync.Success, + Status: Success, LastUpdate: textTime{t}, LastUpdateTs: stampTime{t}, Size: "5GB", @@ -28,7 +26,7 @@ func TestStatus(t *testing.T) { b, err := json.Marshal(m) So(err, ShouldBeNil) //fmt.Println(string(b)) - var m2 webMirrorStatus + var m2 WebMirrorStatus err = json.Unmarshal(b, &m2) So(err, ShouldBeNil) // fmt.Printf("%#v", m2) diff --git a/manager/server.go b/manager/server.go index 05c56bf..5953445 100644 --- a/manager/server.go +++ b/manager/server.go @@ -135,11 +135,11 @@ func (s *Manager) listAllJobs(c *gin.Context) { s.returnErrJSON(c, http.StatusInternalServerError, err) return } - webMirStatusList := []webMirrorStatus{} + webMirStatusList := []WebMirrorStatus{} for _, m := range mirrorStatusList { webMirStatusList = append( webMirStatusList, - convertMirrorStatus(m), + BuildWebMirrorStatus(m), ) } c.JSON(http.StatusOK, webMirStatusList) From 6bca9d2cd531b36cda120ae6ac0cabafb48d5222 Mon Sep 17 00:00:00 2001 From: Yuxiang Zhang Date: Tue, 29 May 2018 19:07:01 +0800 Subject: [PATCH 2/3] fix TestHTTPServer in manager package --- manager/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manager/server_test.go b/manager/server_test.go index c1122b0..bf1a906 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -125,7 +125,7 @@ func TestHTTPServer(t *testing.T) { }) Convey("list all job status of all workers", func(ctx C) { - var ms []webMirrorStatus + var ms []WebMirrorStatus resp, err := GetJSON(baseURL+"/jobs", &ms, nil) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) From 95bb4bbd5e1244465b66e0999c58083fc7b13925 Mon Sep 17 00:00:00 2001 From: Yuxiang Zhang Date: Tue, 29 May 2018 21:16:28 +0800 Subject: [PATCH 3/3] report the last ended time (updated whether successful or not) of jobs --- internal/msg.go | 1 + internal/status_web.go | 4 ++++ internal/status_web_test.go | 34 ++++++++++++++++++++++++++++++++++ manager/db_test.go | 7 +++++-- manager/server.go | 5 +++++ manager/server_test.go | 30 ++++++++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 2 deletions(-) diff --git a/internal/msg.go b/internal/msg.go index 15791b9..c0efb7a 100644 --- a/internal/msg.go +++ b/internal/msg.go @@ -13,6 +13,7 @@ type MirrorStatus struct { IsMaster bool `json:"is_master"` Status SyncStatus `json:"status"` LastUpdate time.Time `json:"last_update"` + LastEnded time.Time `json:"last_ended"` Upstream string `json:"upstream"` Size string `json:"size"` ErrorMsg string `json:"error_msg"` diff --git a/internal/status_web.go b/internal/status_web.go index 33e15ca..9329c96 100644 --- a/internal/status_web.go +++ b/internal/status_web.go @@ -43,6 +43,8 @@ type WebMirrorStatus struct { Status SyncStatus `json:"status"` LastUpdate textTime `json:"last_update"` LastUpdateTs stampTime `json:"last_update_ts"` + LastEnded textTime `json:"last_ended"` + LastEndedTs stampTime `json:"last_ended_ts"` Upstream string `json:"upstream"` Size string `json:"size"` // approximate size } @@ -54,6 +56,8 @@ func BuildWebMirrorStatus(m MirrorStatus) WebMirrorStatus { Status: m.Status, LastUpdate: textTime{m.LastUpdate}, LastUpdateTs: stampTime{m.LastUpdate}, + LastEnded: textTime{m.LastEnded}, + LastEndedTs: stampTime{m.LastEnded}, Upstream: m.Upstream, Size: m.Size, } diff --git a/internal/status_web_test.go b/internal/status_web_test.go index ce31630..97453ed 100644 --- a/internal/status_web_test.go +++ b/internal/status_web_test.go @@ -19,6 +19,8 @@ func TestStatus(t *testing.T) { Status: Success, LastUpdate: textTime{t}, LastUpdateTs: stampTime{t}, + LastEnded: textTime{t}, + LastEndedTs: stampTime{t}, Size: "5GB", Upstream: "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/", } @@ -36,6 +38,38 @@ func TestStatus(t *testing.T) { So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix()) So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) + So(m2.LastEnded.Unix(), ShouldEqual, m.LastEnded.Unix()) + So(m2.LastEndedTs.Unix(), ShouldEqual, m.LastEnded.Unix()) + So(m2.LastEnded.UnixNano(), ShouldEqual, m.LastEnded.UnixNano()) + So(m2.LastEndedTs.UnixNano(), ShouldEqual, m.LastEnded.UnixNano()) + So(m2.Size, ShouldEqual, m.Size) + So(m2.Upstream, ShouldEqual, m.Upstream) + }) + Convey("BuildWebMirrorStatus should work", t, func() { + m := MirrorStatus{ + Name: "arch-sync3", + Worker: "testWorker", + IsMaster: true, + Status: Failed, + LastUpdate: time.Now().Add(-time.Minute * 30), + LastEnded: time.Now(), + Upstream: "mirrors.tuna.tsinghua.edu.cn", + Size: "4GB", + } + + var m2 WebMirrorStatus + m2 = BuildWebMirrorStatus(m) + // fmt.Printf("%#v", m2) + So(m2.Name, ShouldEqual, m.Name) + So(m2.Status, ShouldEqual, m.Status) + So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix()) + So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix()) + So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) + So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano()) + So(m2.LastEnded.Unix(), ShouldEqual, m.LastEnded.Unix()) + So(m2.LastEndedTs.Unix(), ShouldEqual, m.LastEnded.Unix()) + So(m2.LastEnded.UnixNano(), ShouldEqual, m.LastEnded.UnixNano()) + So(m2.LastEndedTs.UnixNano(), ShouldEqual, m.LastEnded.UnixNano()) So(m2.Size, ShouldEqual, m.Size) So(m2.Upstream, ShouldEqual, m.Upstream) }) diff --git a/manager/db_test.go b/manager/db_test.go index 1d13eec..5e6e79d 100644 --- a/manager/db_test.go +++ b/manager/db_test.go @@ -65,6 +65,7 @@ func TestBoltAdapter(t *testing.T) { IsMaster: true, Status: Success, LastUpdate: time.Now(), + LastEnded: time.Now(), Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "3GB", }, @@ -73,7 +74,8 @@ func TestBoltAdapter(t *testing.T) { Worker: testWorkerIDs[1], IsMaster: true, Status: Disabled, - LastUpdate: time.Now(), + LastUpdate: time.Now().Add(-time.Hour), + LastEnded: time.Now(), Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "4GB", }, @@ -82,7 +84,8 @@ func TestBoltAdapter(t *testing.T) { Worker: testWorkerIDs[1], IsMaster: true, Status: Success, - LastUpdate: time.Now(), + LastUpdate: time.Now().Add(-time.Second), + LastEnded: time.Now(), Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "4GB", }, diff --git a/manager/server.go b/manager/server.go index 5953445..2563582 100644 --- a/manager/server.go +++ b/manager/server.go @@ -242,6 +242,11 @@ func (s *Manager) updateJobOfWorker(c *gin.Context) { } else { status.LastUpdate = curStatus.LastUpdate } + if status.Status == Success || status.Status == Failed { + status.LastEnded = time.Now() + } else { + status.LastEnded = curStatus.LastEnded + } // Only message with meaningful size updates the mirror size if len(curStatus.Size) > 0 && curStatus.Size != "unknown" { diff --git a/manager/server_test.go b/manager/server_test.go index bf1a906..27f3cd4 100644 --- a/manager/server_test.go +++ b/manager/server_test.go @@ -121,6 +121,7 @@ func TestHTTPServer(t *testing.T) { So(m.Size, ShouldEqual, status.Size) So(m.IsMaster, ShouldEqual, status.IsMaster) So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second) + So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second) }) @@ -137,6 +138,7 @@ func TestHTTPServer(t *testing.T) { So(m.Size, ShouldEqual, status.Size) So(m.IsMaster, ShouldEqual, status.IsMaster) So(time.Now().Sub(m.LastUpdate.Time), ShouldBeLessThan, 1*time.Second) + So(time.Now().Sub(m.LastEnded.Time), ShouldBeLessThan, 1*time.Second) }) @@ -166,6 +168,7 @@ func TestHTTPServer(t *testing.T) { So(m.Size, ShouldEqual, "5GB") So(m.IsMaster, ShouldEqual, status.IsMaster) So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second) + So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second) }) }) @@ -180,6 +183,32 @@ func TestHTTPServer(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError) }) + + // what if status changed to failed + status.Status = Failed + time.Sleep(3 * time.Second) + resp, err = PostJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", baseURL, status.Worker, status.Name), status, nil) + defer resp.Body.Close() + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + + Convey("What if syncing job failed", func(ctx C) { + var ms []MirrorStatus + resp, err := GetJSON(baseURL+"/workers/test_worker1/jobs", &ms, nil) + + So(err, ShouldBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + // err = json.NewDecoder(resp.Body).Decode(&mirrorStatusList) + m := ms[0] + So(m.Name, ShouldEqual, status.Name) + So(m.Worker, ShouldEqual, status.Worker) + So(m.Status, ShouldEqual, status.Status) + So(m.Upstream, ShouldEqual, status.Upstream) + So(m.Size, ShouldEqual, status.Size) + So(m.IsMaster, ShouldEqual, status.IsMaster) + So(time.Now().Sub(m.LastUpdate), ShouldBeGreaterThan, 3*time.Second) + So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second) + }) }) Convey("update mirror status of an inexisted worker", func(ctx C) { @@ -190,6 +219,7 @@ func TestHTTPServer(t *testing.T) { IsMaster: true, Status: Success, LastUpdate: time.Now(), + LastEnded: time.Now(), Upstream: "mirrors.tuna.tsinghua.edu.cn", Size: "4GB", }