From 85ce9c1270a497b5c775248fbcbbc184e6dd5fc5 Mon Sep 17 00:00:00 2001
From: zyx <z4yx@users.noreply.github.com>
Date: Wed, 3 Jun 2020 19:47:14 +0800
Subject: [PATCH] wait for docker container removal

---
 worker/docker.go      | 24 ++++++++++++++++++++++++
 worker/docker_test.go | 19 ++++++++++++-------
 2 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/worker/docker.go b/worker/docker.go
index 0993417..8f516ac 100644
--- a/worker/docker.go
+++ b/worker/docker.go
@@ -3,6 +3,9 @@ package worker
 import (
 	"fmt"
 	"os"
+	"time"
+
+	"github.com/codeskyblue/go-sh"
 )
 
 type dockerHook struct {
@@ -60,6 +63,27 @@ func (d *dockerHook) postExec() error {
 	// sh.Command(
 	// 	"docker", "rm", "-f", d.Name(),
 	// ).Run()
+	name := d.Name()
+	retry := 10
+	for ; retry > 0; retry-- {
+		out, err := sh.Command(
+			"docker", "ps", "-a",
+			"--filter", "name=^"+name+"$",
+			"--format", "{{.Status}}",
+		).Output()
+		if err != nil {
+			logger.Errorf("docker ps failed: %v", err)
+			break
+		}
+		if len(out) == 0 {
+			break
+		}
+		logger.Debugf("container %s still exists: '%s'", name, string(out))
+		time.Sleep(1 * time.Second)
+	}
+	if retry == 0 {
+		logger.Warningf("container %s not removed automatically, next sync may fail", name)
+	}
 	d.provider.ExitContext()
 	return nil
 }
diff --git a/worker/docker_test.go b/worker/docker_test.go
index 01d6a45..2d1d849 100644
--- a/worker/docker_test.go
+++ b/worker/docker_test.go
@@ -94,22 +94,27 @@ sleep 20
 			}
 			exitedErr <- err
 		}()
-		cmdRun("ps", []string{"aux"})
 
 		// Wait for docker running
-		time.Sleep(8 * time.Second)
-
-		cmdRun("ps", []string{"aux"})
+		for wait := 0; wait < 8; wait++ {
+			names, err := getDockerByName(d.Name())
+			So(err, ShouldBeNil)
+			if names != "" {
+				break
+			}
+			time.Sleep(1 * time.Second)
+		}
+		// cmdRun("ps", []string{"aux"})
 
 		// assert container running
 		names, err := getDockerByName(d.Name())
 		So(err, ShouldBeNil)
-		// So(names, ShouldEqual, d.Name()+"\n")
+		So(names, ShouldEqual, d.Name()+"\n")
 
 		err = provider.Terminate()
-		// So(err, ShouldBeNil)
+		So(err, ShouldBeNil)
 
-		cmdRun("ps", []string{"aux"})
+		// cmdRun("ps", []string{"aux"})
 		<-exitedErr
 
 		// container should be terminated and removed