support simple pattern substitution

apt-sync.py

@@ -3,6 +3,7 @@ import hashlib
 import traceback
 import json
 import os
+import re
 import shutil
 import subprocess as sp
 import tempfile
@@ -16,6 +17,8 @@ OS_TEMPLATE = {
     'debian-current': ["jessie", "stretch", "buster"],
 }
 
+pattern_os_template = re.compile(r"@\{(.+)\}")
+
 apt_download = Path(__file__).parent / "helpers" / "apt-download-binary"
 if not apt_download.is_file():
     raise OSError(f"File not found: {apt_download}")
@@ -28,7 +31,11 @@ def check_args(prop: str, lst: List[str]):
 def replace_os_template(os_list: List[str]) -> List[str]:
     ret = []
     for i in os_list:
-        if i.startswith('@'):
+        matched = pattern_os_template.search(i)
+        if matched:
+            for os in OS_TEMPLATE[matched.group(1)]:
+                ret.append(pattern_os_template.sub(os, i))
+        elif i.startswith('@'):
             ret.extend(OS_TEMPLATE[i[1:]])
         else:
             ret.append(i)
@@ -68,10 +75,10 @@ def main():
                     os, comp, arch,
                     str(args.working_dir.absolute()),
                     filelist[1] ]
-                # print(shell_args)
-                ret = sp.run(shell_args)
-                if ret.returncode != 0:
-                    failed.append((os, comp, arch))
+                print(shell_args)
+                # ret = sp.run(shell_args)
+                # if ret.returncode != 0:
+                #     failed.append((os, comp, arch))
     if len(failed) > 0:
         print("Failed APT repos: ", failed)
     if args.delete:

kubernetes.sh

@@ -12,14 +12,13 @@ BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://packages.cloud.google.com"}
 YUM_PATH="${BASE_PATH}/yum/repos"
 APT_PATH="${BASE_PATH}/apt"
 
-APT_VERSIONS=kubernetes-jessie,kubernetes-stretch,kubernetes-trusty,kubernetes-xenial,minikube-jessie,minikube-trusty
 EL_VERSIONS=(kubernetes-el7-aarch64 kubernetes-el7-armhfp kubernetes-el7-x86_64)
 
 mkdir -p ${YUM_PATH} ${APT_PATH}
 
 
 # =================== APT repos ===============================
-"$apt_sync" "${BASE_URL}/apt" $APT_VERSIONS main amd64,i386,armhf,arm64 "$APT_PATH"
+"$apt_sync" "${BASE_URL}/apt" "kubernetes-@{debian-current},kubernetes-@{ubuntu-lts}" main amd64,i386,armhf,arm64 "$APT_PATH"
 echo "APT finished"
 
 # =================== YUM/DNF repos ==========================

mongodb.sh

@@ -2,18 +2,15 @@
 set -e
 
 _here=`dirname $(realpath $0)`
-. ${_here}/helpers/apt-download
-
-[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1)
+apt_sync="${_here}/apt-sync.py" 
 
 BASE_PATH="${TUNASYNC_WORKING_DIR}"
+BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://repo.mongodb.org"}
 
 YUM_PATH="${BASE_PATH}/yum"
 APT_PATH="${BASE_PATH}/apt"
 
 RHEL_VERSIONS=("6" "7" "8")
-UBUNTU_VERSIONS=("trusty" "precise" "xenial" "bionic")
-DEBIAN_VERSIONS=("wheezy" "jessie" "stretch" "buster")
 MONGO_VERSIONS=("4.2" "4.0" "3.6")
 STABLE_VERSION="4.2"
 
@@ -67,29 +64,12 @@ if [[ ! -z ${DRY_RUN:-} ]]; then
 	export APT_DRY_RUN=1
 fi
 
-base_url="http://repo.mongodb.org/apt/ubuntu"
-for ubver in ${UBUNTU_VERSIONS[@]}; do
+base_url="http://repo.mongodb.org"
 for mgver in ${MONGO_VERSIONS[@]}; do
-		version="$ubver/mongodb-org/$mgver"
-		apt-download-binary ${base_url} "$version" "multiverse" "amd64" "${UBUNTU_PATH}" || true
-		apt-download-binary ${base_url} "$version" "multiverse" "i386" "${UBUNTU_PATH}" || true
+	"$apt_sync" "$BASE_URL/apt/ubuntu" "@{ubuntu-lts}/mongodb-org/$mgver" multiverse amd64,i386 "$UBUNTU_PATH"
+	"$apt_sync" "$BASE_URL/apt/debian" "@{debian-current}/mongodb-org/$mgver" main amd64,i386 "$DEBIAN_PATH"
 done
-	mg_basepath="${UBUNTU_PATH}/dists/$ubver/mongodb-org"
-	[ -e ${mg_basepath}/stable ] || (cd ${mg_basepath}; ln -s ${STABLE_VERSION} stable)
-done
-echo "Ubuntu finished"
-
-base_url="http://repo.mongodb.org/apt/debian"
-for dbver in ${DEBIAN_VERSIONS[@]}; do
-	for mgver in ${MONGO_VERSIONS[@]}; do
-		version="$dbver/mongodb-org/$mgver"
-		apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" || true
-		apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" || true
-	done
-	mg_basepath="${DEBIAN_PATH}/dists/$dbver/mongodb-org"
-	[ -e ${mg_basepath}/stable ] || (cd ${mg_basepath}; ln -s ${STABLE_VERSION} stable)
-done
-echo "Debian finished"
+echo "APT finished"
 
 
 # vim: ts=4 sts=4 sw=4