two_stage_rsync

add extraOptions for stage 1 in two stage rsync
add arch_exclude and arch_include support for two stage rsync
add gitlab-ci dockerfile
add dockerfile

.gitlab-ci.yml

@@ -0,0 +1,21 @@
+stages:
+  - build
+
+build-image-job:
+  stage: build
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  script:
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
+    - >-
+      /kaniko/executor
+      --context "${CI_PROJECT_DIR}"
+      --dockerfile "${CI_PROJECT_DIR}/Dockerfile"
+      --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TITLE}"
+      --build-arg TARGETOS=linux
+      --build-arg TARGETARCH=amd64
+      --force
+  rules:
+    - if: $CI_COMMIT_TITLE

Dockerfile

@@ -0,0 +1,38 @@
+FROM --platform=$TARGETPLATFORM golang:1.13-alpine as builder
+
+RUN apk add git musl-dev gcc make --no-cache
+
+ENV GO111MODULE on
+
+ENV GOPROXY https://goproxy.cn
+
+COPY * /mnt/
+
+COPY .* /mnt/
+
+RUN cd /mnt && git checkout twoStageRsync && make all
+
+RUN cd / && git clone https://github.com/tuna/tunasync-scripts.git
+
+FROM --platform=$TARGETPLATFORM alpine:3
+
+RUN apk update && apk add --no-cache rsync wget htop bash python3 && \
+    mkdir -p /app && mkdir /data && rm -rf /var/cache/apk/* 
+
+RUN wget http://ftp-master.debian.org/ftpsync.tar.gz && tar -vxf *.tar.gz -C / && rm -rf *.tar.gz && cp -rf /distrib/* / && rm -rf /distrib
+
+
+# 使用变量必须申明
+ARG TARGETOS
+
+ARG TARGETARCH
+
+COPY --from=builder /mnt/build-${TARGETOS}-${TARGETARCH}*/* /app/bin/
+
+COPY --from=builder /tunasync-scripts /home/scripts
+
+ENV PATH="${PATH}:/app/bin"
+
+WORKDIR /app
+
+ENTRYPOINT [ "/app/bin/tunasync" ]

worker/config.go

@@ -179,6 +179,8 @@ type mirrorConfig struct {
 	RsyncOptions  []string `toml:"rsync_options"`
 	RsyncOverride []string `toml:"rsync_override"`
 	Stage1Profile string   `toml:"stage1_profile"`
+	ArchInclude   []string `toml:"arch_include"`
+	ArchExclude   []string `toml:"arch_exclude"`
 
 	MemoryLimit MemBytes `toml:"memory_limit"`
 

worker/provider.go

@@ -169,6 +169,8 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 			password:          mirror.Password,
 			excludeFile:       mirror.ExcludeFile,
 			extraOptions:      mirror.RsyncOptions,
+			archInclude:       mirror.ArchInclude,
+			archExclude:       mirror.ArchExclude,
 			rsyncNeverTimeout: mirror.RsyncNoTimeo,
 			rsyncTimeoutValue: mirror.RsyncTimeout,
 			rsyncEnv:          mirror.Env,

worker/two_stage_rsync_provider.go

@@ -14,7 +14,7 @@ type twoStageRsyncConfig struct {
 	rsyncCmd                                     string
 	stage1Profile                                string
 	upstreamURL, username, password, excludeFile string
-	extraOptions                                 []string
+	extraOptions, archInclude, archExclude       []string
 	rsyncNeverTimeout                            bool
 	rsyncTimeoutValue                            int
 	rsyncEnv                                     map[string]string
@@ -31,6 +31,7 @@ type twoStageRsyncProvider struct {
 	twoStageRsyncConfig
 	stage1Options []string
 	stage2Options []string
+	archOptions   []string
 	dataSize      string
 }
 
@@ -43,6 +44,63 @@ var rsyncStage1Profiles = map[string]([]string){
 	},
 }
 
+func genRsyncFiltersForDebianArch(flag string, archList []string) ([]string, error) {
+	archFilters := make([]string, 0)
+	excludeArch := make([]string, 0)
+	allArch := map[string]string{
+		"source":         "",
+		"alpha":          "",
+		"amd64":          "",
+		"arm":            "",
+		"armel":          "",
+		"armhf":          "",
+		"hppa":           "",
+		"hurd-i386":      "",
+		"i386":           "",
+		"ia64":           "",
+		"kfreebsd-amd64": "",
+		"kfreebsd-i386":  "",
+		"mips":           "",
+		"mipsel":         "",
+		"mips64el":       "",
+		"powerpc":        "",
+		"ppc64el":        "",
+		"s390":           "",
+		"s390x":          "",
+		"sparc":          "",
+	}
+	if flag == "exclude" {
+		excludeArch = archList
+	}
+
+	if flag == "include" {
+		for _, arch := range archList {
+			delete(allArch, arch)
+		}
+		for k := range allArch {
+			excludeArch = append(excludeArch, k)
+		}
+	}
+
+	for _, arch := range excludeArch {
+		if arch == "source" {
+			archFilters = append(archFilters, "--exclude=/dists/**/source", "--exclude=/pool/**/*.tar.*", "--exclude=/pool/**/*.diff.*", "--exclude=/pool/**/*.dsc")
+		} else {
+			archFilters = append(archFilters, fmt.Sprintf("--exclude=/dists/**/binary-%s/", arch))
+			archFilters = append(archFilters, fmt.Sprintf("--exclude=/dists/**/installer-%s/", arch))
+			archFilters = append(archFilters, fmt.Sprintf("--exclude=Contents-%s.gz", arch))
+			archFilters = append(archFilters, fmt.Sprintf("--exclude=Contents-udeb-%s.gz", arch))
+			archFilters = append(archFilters, fmt.Sprintf("--exclude=/dists/**/Contents-%s.diff/", arch))
+			archFilters = append(archFilters, fmt.Sprintf("--exclude=arch-%s.files", arch))
+			archFilters = append(archFilters, fmt.Sprintf("--exclude=arch-%s.list.gz", arch))
+			archFilters = append(archFilters, fmt.Sprintf("--exclude=*_%s.deb", arch))
+			archFilters = append(archFilters, fmt.Sprintf("--exclude=*_%s.udeb", arch))
+			archFilters = append(archFilters, fmt.Sprintf("--exclude=*_%s.changes", arch))
+		}
+	}
+	return archFilters, nil
+}
+
 func newTwoStageRsyncProvider(c twoStageRsyncConfig) (*twoStageRsyncProvider, error) {
 	// TODO: check config options
 	if !strings.HasSuffix(c.upstreamURL, "/") {
@@ -52,6 +110,10 @@ func newTwoStageRsyncProvider(c twoStageRsyncConfig) (*twoStageRsyncProvider, er
 		c.retry = defaultMaxRetry
 	}
 
+	if c.archInclude != nil && c.archExclude != nil {
+		return nil, errors.New("ARCH_EXCLUDE and ARCH_INCLUDE are mutually exclusive.  Set only one.")
+	}
+
 	provider := &twoStageRsyncProvider{
 		baseProvider: baseProvider{
 			name:     c.name,
@@ -72,6 +134,20 @@ func newTwoStageRsyncProvider(c twoStageRsyncConfig) (*twoStageRsyncProvider, er
 			"--delete", "--delete-after", "--delay-updates",
 			"--safe-links",
 		},
+		archOptions: []string{},
+	}
+
+	if c.stage1Profile == "debian" || c.stage1Profile == "debian-oldstyle" {
+		if c.archInclude != nil {
+			genedFilters, _ := genRsyncFiltersForDebianArch("include", c.archInclude)
+			provider.archOptions = append(provider.archOptions, genedFilters...)
+
+		}
+		if c.archExclude != nil {
+			genedFilters, _ := genRsyncFiltersForDebianArch("exclude", c.archExclude)
+			provider.archOptions = append(provider.archOptions, genedFilters...)
+		}
+
 	}
 
 	if c.rsyncEnv == nil {
@@ -114,13 +190,16 @@ func (p *twoStageRsyncProvider) Options(stage int) ([]string, error) {
 		if !ok {
 			return nil, errors.New("Invalid Stage 1 Profile")
 		}
-		for _, exc := range stage1Profile {
+		options = append(options, stage1Profile...)
-			options = append(options, exc)
+		options = append(options, p.archOptions...)
+		if p.twoStageRsyncConfig.extraOptions != nil {
+			options = append(options, p.extraOptions...)
 		}
 
 	} else if stage == 2 {
 		options = append(options, p.stage2Options...)
-		if p.extraOptions != nil {
+		options = append(options, p.archOptions...)
+		if p.twoStageRsyncConfig.extraOptions != nil {
 			options = append(options, p.extraOptions...)
 		}
 	} else {