Merge branch 'tuna:master' into adoptium-rhel

anaconda.py

@@ -64,7 +64,7 @@ CONDA_CLOUD_REPOS = (
     "c4aarch64/linux-aarch64", "c4aarch64/noarch",
     "pytorch3d/linux-64", "pytorch3d/noarch",
     "idaholab/linux-64", "idaholab/noarch",
-    "MindSpore/linux-64", "MindSpore/linux-aarch64", "MindSpore/osx-arm64", "MindSpore/osx-64", "MindSpore/win-64",
+    "MindSpore/linux-64", "MindSpore/linux-aarch64", "MindSpore/osx-arm64", "MindSpore/osx-64", "MindSpore/win-64", "MindSpore/noarch",
 )
 
 EXCLUDED_PACKAGES = (
@@ -96,14 +96,26 @@ def md5_check(file: Path, md5: str = None):
             m.update(buf)
     return m.hexdigest() == md5
 
+def sha256_check(file: Path, sha256: str = None):
+    m = hashlib.sha256()
+    with file.open('rb') as f:
+        while True:
+            buf = f.read(1*1024*1024)
+            if not buf:
+                break
+            m.update(buf)
+    return m.hexdigest() == sha256
 
-def curl_download(remote_url: str, dst_file: Path, md5: str = None):
+
+def curl_download(remote_url: str, dst_file: Path, sha256: str = None, md5: str = None):
     sp.check_call([
         "curl", "-o", str(dst_file),
         "-sL", "--remote-time", "--show-error",
         "--fail", "--retry", "10", "--speed-time", "15",
         "--speed-limit", "5000", remote_url,
     ])
+    if sha256 and (not sha256_check(dst_file, sha256)):
+        return "SHA256 mismatch"
     if md5 and (not md5_check(dst_file, md5)):
         return "MD5 mismatch"
 
@@ -141,7 +153,14 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool):
         if meta['name'] in EXCLUDED_PACKAGES:
             continue
 
-        file_size, md5 = meta['size'], meta['md5']
+        file_size = meta['size']
+        # prefer sha256 over md5
+        sha256 = None
+        md5 = None
+        if 'sha256' in meta:
+            sha256 = meta['sha256']
+        elif 'md5' in meta:
+            md5 = meta['md5']
         total_size += file_size
 
         pkg_url = '/'.join([repo_url, filename])
@@ -162,7 +181,7 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool):
         for retry in range(3):
             logging.info("Downloading {}".format(filename))
             try:
-                err = curl_download(pkg_url, dst_file_wip, md5=md5)
+                err = curl_download(pkg_url, dst_file_wip, sha256=sha256, md5=md5)
                 if err is None:
                     dst_file_wip.rename(dst_file)
             except sp.CalledProcessError:
@@ -208,12 +227,12 @@ def sync_installer(repo_url, local_dir: Path):
             if len(tds) != 4:
                 continue
             fname = tds[0].find('a').text
-            md5 = tds[3].text
+            sha256 = tds[3].text
-            if md5 == '<directory>' or len(md5) != 32:
+            if sha256 == '<directory>' or len(sha256) != 64:
                 continue
-            yield (fname, md5)
+            yield (fname, sha256)
 
-    for filename, md5 in remote_list():
+    for filename, sha256 in remote_list():
         pkg_url = "/".join([repo_url, filename])
         dst_file = local_dir / filename
         dst_file_wip = local_dir / ('.downloading.' + filename)
@@ -230,7 +249,7 @@ def sync_installer(repo_url, local_dir: Path):
 
             # Do content verification on ~5% of files (see issue #25)
             if (not len_avail or remote_filesize == local_filesize) and remote_date.timestamp() == local_mtime and \
-                    (random.random() < 0.95 or md5_check(dst_file, md5)):
+                    (random.random() < 0.95 or sha256_check(dst_file, sha256)):
                 logging.info("Skipping {}".format(filename))
 
                 # Stop the scanning if the most recent version is present
@@ -247,7 +266,7 @@ def sync_installer(repo_url, local_dir: Path):
             logging.info("Downloading {}".format(filename))
             err = ''
             try:
-                err = curl_download(pkg_url, dst_file_wip, md5=md5)
+                err = curl_download(pkg_url, dst_file_wip, sha256=sha256)
                 if err is None:
                     dst_file_wip.rename(dst_file)
             except sp.CalledProcessError:

git-recursive.sh

@@ -0,0 +1,157 @@
+#!/bin/bash
+if [[ -z "$TUNASYNC_UPSTREAM_URL" ]];then
+  echo "Please set the TUNASYNC_UPSTREAM_URL"
+  exit 1
+fi
+
+if [[ ! -z "$RECURSIVE" ]];then
+  echo "Sync in a recursive mode"
+fi
+
+TMPDIR=${TMPDIR:-"/tmp"}
+
+MIRROR_BASE_URL=${MIRROR_BASE_URL:-"https://mirrors.tuna.tsinghua.edu.cn/git/"}
+WORKING_DIR_BASE=${WORKING_DIR_BASE:-"/data/mirrors/git/"}
+GENERATED_SCRIPT=${GENERATED_SCRIPT:-"/data/mirrors/git/qemu/qemu.sh"}
+
+if [[ ! -z "$RECURSIVE" ]]; then
+  echo "#!/usr/bin/env bash" > $GENERATED_SCRIPT.tmp
+fi
+
+function script_append() {
+if [[ ! -z "$RECURSIVE" ]]; then
+  echo "$1" >> $GENERATED_SCRIPT.tmp
+fi
+}
+
+depth=0
+
+function echon() {
+  echo depth $depth $@
+}
+
+function repo_init() {
+  local upstream=$1
+  local working_dir=$2
+  git clone --mirror "$upstream" "$working_dir"
+}
+
+function update_linux_git() {
+  local upstream=$1
+  local working_dir=$2
+  cd "$working_dir"
+  echon "==== SYNC $upstream START ===="
+  git remote set-url origin "$upstream"
+  /usr/bin/timeout -s INT 3600 git remote -v update -p
+  local ret=$?
+  [[ $ret -ne 0 ]] && echon "git update failed with rc=$ret"
+  local head=$(git remote show origin | awk '/HEAD branch:/ {print $NF}')
+  [[ -n "$head" ]] && echo "ref: refs/heads/$head" > HEAD
+  objs=$(find objects -type f | wc -l)
+  [[ "$objs" -gt 8 ]] && git repack -a -b -d
+  sz=$(git count-objects -v|grep -Po '(?<=size-pack: )\d+')
+  total_size=$(($total_size+1024*$sz))
+  echon "==== SYNC $upstream DONE ===="
+  return $ret
+}
+
+function git_sync() {
+  local upstream=$1
+  local working_dir=$2
+  if [[ ! -f "$working_dir/HEAD" ]]; then
+    echon "Initializing $upstream mirror"
+    repo_init $1 $2
+    return $?
+  fi
+  update_linux_git $1 $2
+}
+
+function checkout_repo() {
+  local repo_dir="$1"
+  local work_tree="$2"
+  local commit="$3"
+  echon "Checkout $repo_dir to $work_tree"
+  rm -rf $work_tree
+  git clone "$repo_dir" "$work_tree"
+  local ret=$?
+  if [[ $ret -ne 0 ]]; then
+    echon "git clone failed with rc=$ret"
+    return $ret
+  fi
+  if [[ ! -z "$commit" ]]; then
+    echon "Worktree $work_tree switch to commit $commit"
+    git -C $work_tree checkout $commit
+  fi
+
+  local repo_dir_no_git=${repo_dir%%.git}
+  local gitmodules="$work_tree/.gitmodules"
+  if [[ -f "$gitmodules" ]]; then
+    local paths_str=$(cat $gitmodules | grep path | cut -d '=' -f 2 | sed 's/^[[:blank:]]*//')
+    local urls_str=$(cat $gitmodules | grep url | cut -d '=' -f 2 | sed 's/^[[:blank:]]*//')
+    local -a paths
+    local -a urls
+    readarray -t paths <<<"$paths_str"
+    readarray -t urls <<<"$urls_str"
+    local -i i
+    for ((i=0;i<${#paths[@]};i++)); do
+      local path=${paths[$i]}
+      # ignore empty .gitmodules
+      if [[ "$path" == "" ]]; then
+        continue
+      fi
+      local commit=$(git -C $work_tree submodule status $path | cut -d ' ' -f 1 | cut -d '-' -f 2)
+      local git_path=$repo_dir_no_git/$path.git
+      mkdir -p $git_path
+      local mirror_url=$(echo $git_path | sed "s#$WORKING_DIR_BASE#$MIRROR_BASE_URL#")
+      script_append "cat >>.git/config <<EOF"
+      script_append "[submodule \"$path\"]"
+      script_append "	active = true"
+      script_append "	url = $mirror_url"
+      script_append "EOF"
+      script_append "mkdir -p $path"
+      script_append "git clone $mirror_url $path"
+      script_append "git submodule init $path"
+      script_append "git submodule update $path"
+      script_append "pushd $path"
+      git_sync_recursive ${urls[$i]} $git_path $commit
+      script_append "popd"
+    done
+  fi
+}
+
+function git_sync_recursive() {
+  depth=$(($depth+1))
+  local upstream=$1
+  local working_dir=$2
+  # it is ok that commit=""
+  local commit=$3
+  git_sync $1 $2
+  local ret=$?
+  if [[ $ret -ne 0 ]]; then
+    echon "git sync failed with rc=$ret"
+    return $ret
+  fi
+
+  if [[ ! -z "$RECURSIVE" ]]; then
+    working_dir_name=$(basename $2)
+    working_dir_name_no_git=${working_dir_name%%.git}
+    checkout_repo $working_dir $TMPDIR/$working_dir_name_no_git $commit
+  fi
+  depth=$(($depth-1))
+}
+
+path=$(basename $TUNASYNC_WORKING_DIR)
+path_no_git=${path%%.git}
+mirror_url=$(echo $TUNASYNC_WORKING_DIR | sed "s#$WORKING_DIR_BASE#$MIRROR_BASE_URL#")
+script_append "mkdir -p $path_no_git"
+script_append "git clone $mirror_url $path_no_git"
+script_append "pushd $path_no_git"
+git_sync_recursive $TUNASYNC_UPSTREAM_URL $TUNASYNC_WORKING_DIR
+script_append "git submodule absorbgitdirs"
+script_append "popd"
+
+if [[ ! -z "$RECURSIVE" ]]; then
+  mv $GENERATED_SCRIPT.tmp $GENERATED_SCRIPT
+fi
+
+echo "Total size is" $(numfmt --to=iec $total_size)

github-raw.py

@@ -28,8 +28,6 @@ def delete_line_with_gbpdistro(s: str) -> str:
 REPOS = [
     # owner/repo, tree, tree, tree, blob
     ## for stackage
-    ["fpco/minghc", "master", "bin", "7z.exe"],
-    ["fpco/minghc", "master", "bin", "7z.dll"],
     ["fpco/stackage-content", "master", "stack", "global-hints.yaml"],
     ## for rosdep
     { "path": ["ros/rosdistro", "master", "rosdep", "sources.list.d", "20-default.list"], "filter": [ raw_to_mirror, delete_line_with_gbpdistro ] },

github-release.py

@@ -15,6 +15,7 @@ BASE_URL = os.getenv("TUNASYNC_UPSTREAM_URL", "https://api.github.com/repos/")
 WORKING_DIR = os.getenv("TUNASYNC_WORKING_DIR")
 REPOS = [
     "Homebrew/homebrew-portable-ruby",  # Used by homebrew-bottles
+    "Homebrew/glibc-bootstrap",  # Used by homebrew-bottles, see #1586
     {"repo": "googlefonts/noto-fonts", "tarball": True},
     {"repo": "googlefonts/noto-cjk", "tarball": True},
     {"repo": "googlefonts/noto-emoji", "tarball": True},
@@ -41,7 +42,7 @@ REPOS = [
     "conda-forge/miniforge",
     "texstudio-org/texstudio",
     "Stellarium/stellarium",
-    "UNIDY2002/THUInfo",
+    "thu-info-community/thu-info-app",
     {"repo": "obsproject/obs-studio", "versions": 5, "pre_release": False},
     "com-lihaoyi/mill", # better scala build tool,
     "dbeaver/dbeaver", # issue #1348

kde-neon.sh

@@ -12,7 +12,7 @@ APT_PATH="${BASE_PATH}/user"
 export REPO_SIZE_FILE=/tmp/reposize.$RANDOM
 
 # =================== APT repos ===============================
-"$apt_sync" --delete "${BASE_URL}/user" focal,bionic main dep11,cnf,all,amd64,i386 "${BASE_PATH}/user"
+"$apt_sync" --delete "${BASE_URL}/user" jammy,focal,bionic main dep11,cnf,all,amd64,i386 "${BASE_PATH}/user"
 echo "APT finished"
 
 "${_here}/helpers/size-sum.sh" $REPO_SIZE_FILE --rm

mongodb.sh

@@ -8,7 +8,7 @@ yum_sync="${_here}/yum-sync.py"
 BASE_PATH="${TUNASYNC_WORKING_DIR}"
 BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://repo.mongodb.org"}
 
-MONGO_VERSIONS=("5.0" "4.4" "4.2")
+MONGO_VERSIONS=("6.0" "5.0" "4.4" "4.2")
 STABLE_VERSION="5.0"
 
 YUM_PATH="${BASE_PATH}/yum"

nixos-images.py

@@ -99,7 +99,7 @@ def get_url(name):
     return response.headers['x-amz-website-redirect-location']
 
 def clone_images():
-    DOWNLOAD_MATCH = r'nixos-\d\d.\d\d/latest-nixos-\w+-\w+-linux.\w+(.sha256)?'
+    DOWNLOAD_MATCH = r'nixos-(\d\d.\d\d|unstable)/latest-nixos-\w+-\w+-linux.\w+(.sha256)?'
 
     object_names = [
         x.object_name

proxmox.sh

@@ -10,10 +10,16 @@ BASE_URL="${TUNASYNC_UPSTREAM_URL:-"http://download.proxmox.com"}"
 BASE_PATH="${TUNASYNC_WORKING_DIR}"
 
 APT_PATH="${BASE_PATH}/debian"
+PBS_PATH="${BASE_PATH}/pbs"
+PBS_CLIENT_PATH="${BASE_PATH}/pbs-client"
+PMG_PATH="${BASE_PATH}/pmg"
 
 # === download deb packages ====
 
 "$apt_sync" --delete "${BASE_URL}/debian" @debian-current pve-no-subscription,pvetest amd64 "$APT_PATH"
+"$apt_sync" --delete "${BASE_URL}/debian/pbs" @debian-current pbs-no-subscription amd64 "$PBS_PATH"
+"$apt_sync" --delete "${BASE_URL}/debian/pbs-client" @debian-current main amd64 "$PBS_CLIENT_PATH"
+"$apt_sync" --delete "${BASE_URL}/debian/pmg" @debian-current pmg-no-subscription amd64 "$PMG_PATH"
 echo "Debian finished"
 
 # === download standalone files ====

pypi.sh

@@ -24,6 +24,7 @@ if [[ $INIT == "0" ]]; then
 	cat << EOF
 [mirror]
 directory = ${TUNASYNC_WORKING_DIR}
+storage-backend = filesystem
 master = ${PYPI_MASTER}
 ${DOWNLOAD_MIRROR}
 json = true

s3.sh

@@ -8,5 +8,5 @@ fi
 
 [[ ! -d "${TUNASYNC_WORKING_DIR}" ]] && mkdir -p "${TUNASYNC_WORKING_DIR}"
 mkdir /tmp/none; cd /tmp/none # enter an empty folder, so the stars in TUNASYNC_AWS_OPTIONS are not expanded
-exec aws --no-sign-request ${ENDPOINT} s3 sync ${TUNASYNC_AWS_OPTIONS} "${TUNASYNC_UPSTREAM_URL}" "${TUNASYNC_WORKING_DIR}"
+exec aws --no-sign-request ${ENDPOINT} s3 sync --exact-timestamps ${TUNASYNC_AWS_OPTIONS} "${TUNASYNC_UPSTREAM_URL}" "${TUNASYNC_WORKING_DIR}"
 

stackage.py

@@ -10,6 +10,8 @@ import shutil
 import subprocess
 import yaml
 
+MIRROR_BASE_URL = os.getenv("MIRROR_BASE_URL", 'https://mirrors.tuna.tsinghua.edu.cn/stackage')
+MIRROR_GITHUB_RELEASE_URL = os.getenv("MIRROR_GITHUB_RELEASE_URL", 'https://mirrors.tuna.tsinghua.edu.cn/github-release')
 
 class StackageSession(object):
     def __init__(self):
@@ -38,7 +40,8 @@ class StackageSession(object):
         d = yaml.load(
             requests
                 .get('https://raw.githubusercontent.com/commercialhaskell/stackage-content/master/stack/stack-setup-2.yaml')
-                .content
+                .content,
+            Loader=yaml.FullLoader
         )
         for platform in d['ghc']:
             for ver in d['ghc'][platform]:
@@ -48,27 +51,29 @@ class StackageSession(object):
                     d['ghc'][platform][ver]['sha1'],
                 )
                 d['ghc'][platform][ver]['url'] = (
-                    'http://mirrors.tuna.tsinghua.edu.cn/stackage/ghc/{}'
+                    '{}/ghc/{}'
-                    .format(d['ghc'][platform][ver]['url'].split('/')[-1])
+                    .format(MIRROR_BASE_URL, d['ghc'][platform][ver]['url'].split('/')[-1])
                 )
 
         if 'msys2' in d:
             if 'windows32' in d['msys2']:
-                print('windows32')
                 d['msys2']['windows32']['url'] = d['msys2']['windows32']['url'].replace(
                     'https://github.com/fpco/stackage-content/releases/download/',
-                    'https://mirrors.tuna.tsinghua.edu.cn/github-release/commercialhaskell/stackage-content/msys2-')
+                    f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/msys2-')
             if 'windows64' in d['msys2']:
-                print('windows64')
                 d['msys2']['windows64']['url'] = d['msys2']['windows64']['url'].replace(
                     'https://github.com/commercialhaskell/stackage-content/releases/download/',
-                    'https://mirrors.tuna.tsinghua.edu.cn/github-release/commercialhaskell/stackage-content/')
+                    f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/')
 
         if 'sevenzexe-info' in d:
-            d['sevenzexe-info']['url'] = 'https://mirrors.tuna.tsinghua.edu.cn/github-raw/fpco/minghc/master/bin/7z.exe'
+            d['sevenzexe-info']['url'] = d['sevenzexe-info']['url'].replace(
+                'https://github.com/commercialhaskell/stackage-content/releases/download/',
+                f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/')
 
         if 'sevenzdll-info' in d:
-            d['sevenzdll-info']['url'] = 'https://mirrors.tuna.tsinghua.edu.cn/github-raw/fpco/minghc/master/bin/7z.dll'
+            d['sevenzdll-info']['url'] = d['sevenzdll-info']['url'].replace(
+                'https://github.com/commercialhaskell/stackage-content/releases/download/',
+                f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/')
 
         for i in ['portable-git', 'stack', 'ghcjs']:
             del d[i]

virtualbox.sh

@@ -32,7 +32,7 @@ echo "Debian and ubuntu finished"
 timeout -s INT 30 wget ${WGET_OPTIONS:-} -q -O "/tmp/index.html" "${BASE_URL}/"
 timeout -s INT 30 wget ${WGET_OPTIONS:-} -q -O "${BASE_PATH}/LATEST.TXT" "${BASE_URL}/LATEST.TXT"
 
-for((major=4;major<=6;major++));do
+for((major=4;major<=7;major++));do
 	LATEST_VERSION=$(grep -P -o "\"$major\.[\\d\\.]+/\"" -r /tmp/index.html|tail -n 1)
 	LATEST_VERSION=${LATEST_VERSION%/\"}
 	LATEST_VERSION=${LATEST_VERSION#\"}
@@ -89,10 +89,10 @@ LATEST_VERSION=`cat "${BASE_PATH}/LATEST.TXT"`
 for filename in ${BASE_PATH}/${LATEST_VERSION}/*.*; do
 	case $filename in
 		*Win.exe)
-			ln -sf "${filename}" "${BASE_PATH}/virtualbox-Win-latest.exe"
+			ln -sf "${filename#"$BASE_PATH/"}" "${BASE_PATH}/virtualbox-Win-latest.exe"
 			;;
 		*OSX.dmg)
-			ln -sf "${filename}" "${BASE_PATH}/virtualbox-osx-latest.dmg"
+			ln -sf "${filename#"$BASE_PATH/"}" "${BASE_PATH}/virtualbox-osx-latest.dmg"
 			;;
 	esac
 done

wine-builds.sh

@@ -10,10 +10,10 @@ BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://dl.winehq.org/wine-builds"}
 
 export REPO_SIZE_FILE=/tmp/reposize.$RANDOM
 
-"$apt_sync" --delete "$BASE_URL/ubuntu" @ubuntu-lts main amd64,i386 "$BASE_PATH/ubuntu"
+"$apt_sync" --delete "$BASE_URL/ubuntu" @ubuntu-lts main amd64,i386,all "$BASE_PATH/ubuntu"
 echo "APT for Ubuntu finished"
 
-"$apt_sync" --delete "$BASE_URL/debian" @debian-latest2 main amd64,i386 "$BASE_PATH/debian"
+"$apt_sync" --delete "$BASE_URL/debian" @debian-latest2 main amd64,i386,all "$BASE_PATH/debian"
 echo "APT for Debian finished"
 
 echo "APT finished"