diff --git a/worker/job.go b/worker/job.go index 1f89130..98133e4 100644 --- a/worker/job.go +++ b/worker/job.go @@ -180,7 +180,6 @@ func (m *mirrorJob) Run(managerChan chan<- jobMessage, semaphore chan empty) err logger.Debug("syncing done") case <-time.After(timeout): logger.Notice("provider timeout") - stopASAP = true termErr = provider.Terminate() syncErr = fmt.Errorf("%s timeout after %v", m.Name(), timeout) case <-kill: diff --git a/worker/job_test.go b/worker/job_test.go index fded5fe..fc9459d 100644 --- a/worker/job_test.go +++ b/worker/job_test.go @@ -335,7 +335,6 @@ echo $TUNASYNC_WORKING_DIR }) }) - Convey("When a job timed out", func(ctx C) { scriptContent := `#!/bin/bash echo $TUNASYNC_WORKING_DIR @@ -371,6 +370,30 @@ echo $TUNASYNC_WORKING_DIR job.ctrlChan <- jobDisable <-job.disabled }) + + Convey("It should be retried", func(ctx C) { + go job.Run(managerChan, semaphore) + job.ctrlChan <- jobStart + time.Sleep(1 * time.Second) + msg := <-managerChan + So(msg.status, ShouldEqual, PreSyncing) + + for i := 0; i < defaultMaxRetry; i++ { + msg = <-managerChan + So(msg.status, ShouldEqual, Syncing) + + job.ctrlChan <- jobStart // should be ignored + + msg = <-managerChan + So(msg.status, ShouldEqual, Failed) + So(msg.msg, ShouldContainSubstring, "timeout after") + // re-schedule after last try + So(msg.schedule, ShouldEqual, i == defaultMaxRetry-1) + } + + job.ctrlChan <- jobDisable + <-job.disabled + }) }) })