apt sync scripts refactoring

apt-sync.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+import hashlib
+import traceback
+import json
+import os
+import shutil
+import subprocess as sp
+import tempfile
+import argparse
+from pathlib import Path
+from typing import List
+
+OS_TEMPLATE = {
+    'ubuntu-current': ["trusty", "xenial", "bionic", "eoan"],
+    'ubuntu-lts': ["trusty", "xenial", "bionic"],
+    'debian-current': ["jessie", "stretch", "buster"],
+}
+
+apt_download = Path(__file__).parent / "helpers" / "apt-download-binary"
+if not apt_download.is_file():
+    raise OSError(f"File not found: {apt_download}")
+
+def check_args(prop: str, lst: List[str]):
+    for s in lst:
+        if len(s)==0 or ' ' in s:
+            raise ValueError(f"Invalid item in {prop}: {repr(s)}")
+
+def replace_os_template(os_list: List[str]) -> List[str]:
+    ret = []
+    for i in os_list:
+        if i.startswith('@'):
+            ret.extend(OS_TEMPLATE[i[1:]])
+        else:
+            ret.append(i)
+    return ret
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("base_url", type=str, help="base URL")
+    parser.add_argument("os_version", type=str, help="e.g. buster,@ubuntu-lts")
+    parser.add_argument("component", type=str, help="e.g. multiverse,contrib")
+    parser.add_argument("arch", type=str, help="e.g. i386,amd64")
+    parser.add_argument("working_dir", type=Path, help="working directory")
+    parser.add_argument("--delete", action='store_true',
+                        help='delete unreferenced package files')
+    args = parser.parse_args()
+
+    os_list = args.os_version.split(',')
+    check_args("os_version", os_list)
+    component_list = args.component.split(',')
+    check_args("component", component_list)
+    arch_list = args.arch.split(',')
+    check_args("arch", arch_list)
+
+    os_list = replace_os_template(os_list)
+
+    args.working_dir.mkdir(parents=True, exist_ok=True)
+    filelist = tempfile.mkstemp()
+
+    for os in os_list:
+        for comp in component_list:
+            for arch in arch_list:
+                shell_args = [
+                    str(apt_download.absolute()),
+                    args.base_url,
+                    os, comp, arch,
+                    str(args.working_dir.absolute()),
+                    filelist[1] ]
+                print(shell_args)
+                sp.run(shell_args)
+    if args.delete:
+        pass #TODO
+
+if __name__ == "__main__":
+    main()

helpers/apt-download-binary

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+_here=`dirname $(realpath $0)`
+. ${_here}/apt-download
+
+exec apt-download-binary $@

percona.sh

@@ -4,9 +4,7 @@ set -e
 set -o pipefail
 
 _here=`dirname $(realpath $0)`
-. ${_here}/helpers/apt-download
+alias apt-sync="${_here}/apt-sync.py" 
-
-[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1)
 
 BASE_PATH="${TUNASYNC_WORKING_DIR}"
 BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://repo.percona.com"}
@@ -14,25 +12,14 @@ BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://repo.percona.com"}
 YUM_PATH="${BASE_PATH}/yum"
 APT_PATH="${BASE_PATH}/apt"
 
-APT_VERSIONS=("wheezy" "jessie" "trusty" "xenial" "stretch" "bionic" "buster")
 EL_VERSIONS=("6" "7")
 
-mkdir -p ${YUM_PATH} ${APT_PATH}
-
-
 # =================== APT repos ===============================
-if [[ ! -z ${DRY_RUN:-} ]]; then
+apt-sync "${BASE_URL}/apt" @debian-current,@ubuntu-lts main amd64,i386 "${APT_PATH}"
-	export APT_DRY_RUN=1
-fi
-base_url="${BASE_URL}/apt"
-for version in ${APT_VERSIONS[@]}; do
-	for arch in "amd64" "i386"; do
-		apt-download-binary ${base_url} "$version" "main" "$arch" "${APT_PATH}" || true
-	done
-done
 echo "APT finished"
 
 # =================== YUM/DNF repos ==========================
+mkdir -p ${YUM_PATH}
 
 cache_dir="/tmp/yum-percona-cache/"
 cfg="/tmp/yum-percona.conf"