diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..5667a53 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,21 @@ +stages: + - build + +build-image-job: + stage: build + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + script: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json + - >- + /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TITLE}" + --build-arg TARGETOS=linux + --build-arg TARGETARCH=amd64 + --force + rules: + - if: $CI_COMMIT_TITLE diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b93e0a6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +FROM --platform=$TARGETPLATFORM golang:1.13-alpine as builder + +RUN apk add git musl-dev gcc make --no-cache + +ENV GO111MODULE on + +ENV GOPROXY https://goproxy.cn + +COPY * /mnt/ + +COPY .* /mnt/ + +RUN cd /mnt && git checkout twoStageRsync && make all + +RUN cd / && git clone https://github.com/tuna/tunasync-scripts.git + +FROM --platform=$TARGETPLATFORM alpine:3 + +RUN apk update && apk add --no-cache rsync wget htop bash python3 && \ + mkdir -p /app && mkdir /data && rm -rf /var/cache/apk/* + +RUN wget http://ftp-master.debian.org/ftpsync.tar.gz && tar -vxf *.tar.gz -C / && rm -rf *.tar.gz && cp -rf /distrib/* / && rm -rf /distrib + + +# 使用变量必须申明 +ARG TARGETOS + +ARG TARGETARCH + +COPY --from=builder /mnt/build-${TARGETOS}-${TARGETARCH}*/* /app/bin/ + +COPY --from=builder /tunasync-scripts /home/scripts + +ENV PATH="${PATH}:/app/bin" + +WORKDIR /app + +ENTRYPOINT [ "/app/bin/tunasync" ] \ No newline at end of file diff --git a/worker/config.go b/worker/config.go index c91cbc2..ba06a1c 100644 --- a/worker/config.go +++ b/worker/config.go @@ -179,6 +179,8 @@ type mirrorConfig struct { RsyncOptions []string `toml:"rsync_options"` RsyncOverride []string `toml:"rsync_override"` Stage1Profile string `toml:"stage1_profile"` + ArchInclude []string `toml:"arch_include"` + ArchExclude []string `toml:"arch_exclude"` MemoryLimit MemBytes `toml:"memory_limit"` diff --git a/worker/provider.go b/worker/provider.go index 443c8f5..9766dd7 100644 --- a/worker/provider.go +++ b/worker/provider.go @@ -169,6 +169,8 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider { password: mirror.Password, excludeFile: mirror.ExcludeFile, extraOptions: mirror.RsyncOptions, + archInclude: mirror.ArchInclude, + archExclude: mirror.ArchExclude, rsyncNeverTimeout: mirror.RsyncNoTimeo, rsyncTimeoutValue: mirror.RsyncTimeout, rsyncEnv: mirror.Env, diff --git a/worker/two_stage_rsync_provider.go b/worker/two_stage_rsync_provider.go index 7063991..e488a75 100644 --- a/worker/two_stage_rsync_provider.go +++ b/worker/two_stage_rsync_provider.go @@ -14,7 +14,7 @@ type twoStageRsyncConfig struct { rsyncCmd string stage1Profile string upstreamURL, username, password, excludeFile string - extraOptions []string + extraOptions, archInclude, archExclude []string rsyncNeverTimeout bool rsyncTimeoutValue int rsyncEnv map[string]string @@ -31,6 +31,7 @@ type twoStageRsyncProvider struct { twoStageRsyncConfig stage1Options []string stage2Options []string + archOptions []string dataSize string } @@ -43,6 +44,63 @@ var rsyncStage1Profiles = map[string]([]string){ }, } +func genRsyncFiltersForDebianArch(flag string, archList []string) ([]string, error) { + archFilters := make([]string, 0) + excludeArch := make([]string, 0) + allArch := map[string]string{ + "source": "", + "alpha": "", + "amd64": "", + "arm": "", + "armel": "", + "armhf": "", + "hppa": "", + "hurd-i386": "", + "i386": "", + "ia64": "", + "kfreebsd-amd64": "", + "kfreebsd-i386": "", + "mips": "", + "mipsel": "", + "mips64el": "", + "powerpc": "", + "ppc64el": "", + "s390": "", + "s390x": "", + "sparc": "", + } + if flag == "exclude" { + excludeArch = archList + } + + if flag == "include" { + for _, arch := range archList { + delete(allArch, arch) + } + for k := range allArch { + excludeArch = append(excludeArch, k) + } + } + + for _, arch := range excludeArch { + if arch == "source" { + archFilters = append(archFilters, "--exclude=/dists/**/source", "--exclude=/pool/**/*.tar.*", "--exclude=/pool/**/*.diff.*", "--exclude=/pool/**/*.dsc") + } else { + archFilters = append(archFilters, fmt.Sprintf("--exclude=/dists/**/binary-%s/", arch)) + archFilters = append(archFilters, fmt.Sprintf("--exclude=/dists/**/installer-%s/", arch)) + archFilters = append(archFilters, fmt.Sprintf("--exclude=Contents-%s.gz", arch)) + archFilters = append(archFilters, fmt.Sprintf("--exclude=Contents-udeb-%s.gz", arch)) + archFilters = append(archFilters, fmt.Sprintf("--exclude=/dists/**/Contents-%s.diff/", arch)) + archFilters = append(archFilters, fmt.Sprintf("--exclude=arch-%s.files", arch)) + archFilters = append(archFilters, fmt.Sprintf("--exclude=arch-%s.list.gz", arch)) + archFilters = append(archFilters, fmt.Sprintf("--exclude=*_%s.deb", arch)) + archFilters = append(archFilters, fmt.Sprintf("--exclude=*_%s.udeb", arch)) + archFilters = append(archFilters, fmt.Sprintf("--exclude=*_%s.changes", arch)) + } + } + return archFilters, nil +} + func newTwoStageRsyncProvider(c twoStageRsyncConfig) (*twoStageRsyncProvider, error) { // TODO: check config options if !strings.HasSuffix(c.upstreamURL, "/") { @@ -52,6 +110,10 @@ func newTwoStageRsyncProvider(c twoStageRsyncConfig) (*twoStageRsyncProvider, er c.retry = defaultMaxRetry } + if c.archInclude != nil && c.archExclude != nil { + return nil, errors.New("ARCH_EXCLUDE and ARCH_INCLUDE are mutually exclusive. Set only one.") + } + provider := &twoStageRsyncProvider{ baseProvider: baseProvider{ name: c.name, @@ -72,6 +134,20 @@ func newTwoStageRsyncProvider(c twoStageRsyncConfig) (*twoStageRsyncProvider, er "--delete", "--delete-after", "--delay-updates", "--safe-links", }, + archOptions: []string{}, + } + + if c.stage1Profile == "debian" || c.stage1Profile == "debian-oldstyle" { + if c.archInclude != nil { + genedFilters, _ := genRsyncFiltersForDebianArch("include", c.archInclude) + provider.archOptions = append(provider.archOptions, genedFilters...) + + } + if c.archExclude != nil { + genedFilters, _ := genRsyncFiltersForDebianArch("exclude", c.archExclude) + provider.archOptions = append(provider.archOptions, genedFilters...) + } + } if c.rsyncEnv == nil { @@ -114,13 +190,16 @@ func (p *twoStageRsyncProvider) Options(stage int) ([]string, error) { if !ok { return nil, errors.New("Invalid Stage 1 Profile") } - for _, exc := range stage1Profile { - options = append(options, exc) + options = append(options, stage1Profile...) + options = append(options, p.archOptions...) + if p.twoStageRsyncConfig.extraOptions != nil { + options = append(options, p.extraOptions...) } } else if stage == 2 { options = append(options, p.stage2Options...) - if p.extraOptions != nil { + options = append(options, p.archOptions...) + if p.twoStageRsyncConfig.extraOptions != nil { options = append(options, p.extraOptions...) } } else {