diff --git a/anaconda.py b/anaconda.py index bafc8b1..6a86eac 100755 --- a/anaconda.py +++ b/anaconda.py @@ -64,7 +64,7 @@ CONDA_CLOUD_REPOS = ( "c4aarch64/linux-aarch64", "c4aarch64/noarch", "pytorch3d/linux-64", "pytorch3d/noarch", "idaholab/linux-64", "idaholab/noarch", - "MindSpore/linux-64", "MindSpore/linux-aarch64", "MindSpore/osx-arm64", "MindSpore/osx-64", "MindSpore/win-64", + "MindSpore/linux-64", "MindSpore/linux-aarch64", "MindSpore/osx-arm64", "MindSpore/osx-64", "MindSpore/win-64", "MindSpore/noarch", ) EXCLUDED_PACKAGES = ( @@ -96,14 +96,26 @@ def md5_check(file: Path, md5: str = None): m.update(buf) return m.hexdigest() == md5 +def sha256_check(file: Path, sha256: str = None): + m = hashlib.sha256() + with file.open('rb') as f: + while True: + buf = f.read(1*1024*1024) + if not buf: + break + m.update(buf) + return m.hexdigest() == sha256 -def curl_download(remote_url: str, dst_file: Path, md5: str = None): + +def curl_download(remote_url: str, dst_file: Path, sha256: str = None, md5: str = None): sp.check_call([ "curl", "-o", str(dst_file), "-sL", "--remote-time", "--show-error", "--fail", "--retry", "10", "--speed-time", "15", "--speed-limit", "5000", remote_url, ]) + if sha256 and (not sha256_check(dst_file, sha256)): + return "SHA256 mismatch" if md5 and (not md5_check(dst_file, md5)): return "MD5 mismatch" @@ -141,7 +153,14 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool): if meta['name'] in EXCLUDED_PACKAGES: continue - file_size, md5 = meta['size'], meta['md5'] + file_size = meta['size'] + # prefer sha256 over md5 + sha256 = None + md5 = None + if 'sha256' in meta: + sha256 = meta['sha256'] + elif 'md5' in meta: + md5 = meta['md5'] total_size += file_size pkg_url = '/'.join([repo_url, filename]) @@ -162,7 +181,7 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool): for retry in range(3): logging.info("Downloading {}".format(filename)) try: - err = curl_download(pkg_url, dst_file_wip, md5=md5) + err = curl_download(pkg_url, dst_file_wip, sha256=sha256, md5=md5) if err is None: dst_file_wip.rename(dst_file) except sp.CalledProcessError: @@ -208,12 +227,12 @@ def sync_installer(repo_url, local_dir: Path): if len(tds) != 4: continue fname = tds[0].find('a').text - md5 = tds[3].text - if md5 == '' or len(md5) != 32: + sha256 = tds[3].text + if sha256 == '' or len(sha256) != 64: continue - yield (fname, md5) + yield (fname, sha256) - for filename, md5 in remote_list(): + for filename, sha256 in remote_list(): pkg_url = "/".join([repo_url, filename]) dst_file = local_dir / filename dst_file_wip = local_dir / ('.downloading.' + filename) @@ -230,7 +249,7 @@ def sync_installer(repo_url, local_dir: Path): # Do content verification on ~5% of files (see issue #25) if (not len_avail or remote_filesize == local_filesize) and remote_date.timestamp() == local_mtime and \ - (random.random() < 0.95 or md5_check(dst_file, md5)): + (random.random() < 0.95 or sha256_check(dst_file, sha256)): logging.info("Skipping {}".format(filename)) # Stop the scanning if the most recent version is present @@ -247,7 +266,7 @@ def sync_installer(repo_url, local_dir: Path): logging.info("Downloading {}".format(filename)) err = '' try: - err = curl_download(pkg_url, dst_file_wip, md5=md5) + err = curl_download(pkg_url, dst_file_wip, sha256=sha256) if err is None: dst_file_wip.rename(dst_file) except sp.CalledProcessError: diff --git a/git-recursive.sh b/git-recursive.sh new file mode 100755 index 0000000..ff6ff6d --- /dev/null +++ b/git-recursive.sh @@ -0,0 +1,157 @@ +#!/bin/bash +if [[ -z "$TUNASYNC_UPSTREAM_URL" ]];then + echo "Please set the TUNASYNC_UPSTREAM_URL" + exit 1 +fi + +if [[ ! -z "$RECURSIVE" ]];then + echo "Sync in a recursive mode" +fi + +TMPDIR=${TMPDIR:-"/tmp"} + +MIRROR_BASE_URL=${MIRROR_BASE_URL:-"https://mirrors.tuna.tsinghua.edu.cn/git/"} +WORKING_DIR_BASE=${WORKING_DIR_BASE:-"/data/mirrors/git/"} +GENERATED_SCRIPT=${GENERATED_SCRIPT:-"/data/mirrors/git/qemu/qemu.sh"} + +if [[ ! -z "$RECURSIVE" ]]; then + echo "#!/usr/bin/env bash" > $GENERATED_SCRIPT.tmp +fi + +function script_append() { +if [[ ! -z "$RECURSIVE" ]]; then + echo "$1" >> $GENERATED_SCRIPT.tmp +fi +} + +depth=0 + +function echon() { + echo depth $depth $@ +} + +function repo_init() { + local upstream=$1 + local working_dir=$2 + git clone --mirror "$upstream" "$working_dir" +} + +function update_linux_git() { + local upstream=$1 + local working_dir=$2 + cd "$working_dir" + echon "==== SYNC $upstream START ====" + git remote set-url origin "$upstream" + /usr/bin/timeout -s INT 3600 git remote -v update -p + local ret=$? + [[ $ret -ne 0 ]] && echon "git update failed with rc=$ret" + local head=$(git remote show origin | awk '/HEAD branch:/ {print $NF}') + [[ -n "$head" ]] && echo "ref: refs/heads/$head" > HEAD + objs=$(find objects -type f | wc -l) + [[ "$objs" -gt 8 ]] && git repack -a -b -d + sz=$(git count-objects -v|grep -Po '(?<=size-pack: )\d+') + total_size=$(($total_size+1024*$sz)) + echon "==== SYNC $upstream DONE ====" + return $ret +} + +function git_sync() { + local upstream=$1 + local working_dir=$2 + if [[ ! -f "$working_dir/HEAD" ]]; then + echon "Initializing $upstream mirror" + repo_init $1 $2 + return $? + fi + update_linux_git $1 $2 +} + +function checkout_repo() { + local repo_dir="$1" + local work_tree="$2" + local commit="$3" + echon "Checkout $repo_dir to $work_tree" + rm -rf $work_tree + git clone "$repo_dir" "$work_tree" + local ret=$? + if [[ $ret -ne 0 ]]; then + echon "git clone failed with rc=$ret" + return $ret + fi + if [[ ! -z "$commit" ]]; then + echon "Worktree $work_tree switch to commit $commit" + git -C $work_tree checkout $commit + fi + + local repo_dir_no_git=${repo_dir%%.git} + local gitmodules="$work_tree/.gitmodules" + if [[ -f "$gitmodules" ]]; then + local paths_str=$(cat $gitmodules | grep path | cut -d '=' -f 2 | sed 's/^[[:blank:]]*//') + local urls_str=$(cat $gitmodules | grep url | cut -d '=' -f 2 | sed 's/^[[:blank:]]*//') + local -a paths + local -a urls + readarray -t paths <<<"$paths_str" + readarray -t urls <<<"$urls_str" + local -i i + for ((i=0;i<${#paths[@]};i++)); do + local path=${paths[$i]} + # ignore empty .gitmodules + if [[ "$path" == "" ]]; then + continue + fi + local commit=$(git -C $work_tree submodule status $path | cut -d ' ' -f 1 | cut -d '-' -f 2) + local git_path=$repo_dir_no_git/$path.git + mkdir -p $git_path + local mirror_url=$(echo $git_path | sed "s#$WORKING_DIR_BASE#$MIRROR_BASE_URL#") + script_append "cat >>.git/config < str: REPOS = [ # owner/repo, tree, tree, tree, blob ## for stackage - ["fpco/minghc", "master", "bin", "7z.exe"], - ["fpco/minghc", "master", "bin", "7z.dll"], ["fpco/stackage-content", "master", "stack", "global-hints.yaml"], ## for rosdep { "path": ["ros/rosdistro", "master", "rosdep", "sources.list.d", "20-default.list"], "filter": [ raw_to_mirror, delete_line_with_gbpdistro ] }, diff --git a/github-release.py b/github-release.py index 59db9b2..e7fa612 100755 --- a/github-release.py +++ b/github-release.py @@ -15,6 +15,7 @@ BASE_URL = os.getenv("TUNASYNC_UPSTREAM_URL", "https://api.github.com/repos/") WORKING_DIR = os.getenv("TUNASYNC_WORKING_DIR") REPOS = [ "Homebrew/homebrew-portable-ruby", # Used by homebrew-bottles + "Homebrew/glibc-bootstrap", # Used by homebrew-bottles, see #1586 {"repo": "googlefonts/noto-fonts", "tarball": True}, {"repo": "googlefonts/noto-cjk", "tarball": True}, {"repo": "googlefonts/noto-emoji", "tarball": True}, @@ -41,7 +42,7 @@ REPOS = [ "conda-forge/miniforge", "texstudio-org/texstudio", "Stellarium/stellarium", - "UNIDY2002/THUInfo", + "thu-info-community/thu-info-app", {"repo": "obsproject/obs-studio", "versions": 5, "pre_release": False}, "com-lihaoyi/mill", # better scala build tool, "dbeaver/dbeaver", # issue #1348 diff --git a/kde-neon.sh b/kde-neon.sh index 062c2a5..483d415 100755 --- a/kde-neon.sh +++ b/kde-neon.sh @@ -12,7 +12,7 @@ APT_PATH="${BASE_PATH}/user" export REPO_SIZE_FILE=/tmp/reposize.$RANDOM # =================== APT repos =============================== -"$apt_sync" --delete "${BASE_URL}/user" focal,bionic main dep11,cnf,all,amd64,i386 "${BASE_PATH}/user" +"$apt_sync" --delete "${BASE_URL}/user" jammy,focal,bionic main dep11,cnf,all,amd64,i386 "${BASE_PATH}/user" echo "APT finished" "${_here}/helpers/size-sum.sh" $REPO_SIZE_FILE --rm diff --git a/mongodb.sh b/mongodb.sh index 4f2710c..3a64ef1 100755 --- a/mongodb.sh +++ b/mongodb.sh @@ -8,7 +8,7 @@ yum_sync="${_here}/yum-sync.py" BASE_PATH="${TUNASYNC_WORKING_DIR}" BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://repo.mongodb.org"} -MONGO_VERSIONS=("5.0" "4.4" "4.2") +MONGO_VERSIONS=("6.0" "5.0" "4.4" "4.2") STABLE_VERSION="5.0" YUM_PATH="${BASE_PATH}/yum" diff --git a/nixos-images.py b/nixos-images.py index 9e9a8d0..532227b 100755 --- a/nixos-images.py +++ b/nixos-images.py @@ -99,7 +99,7 @@ def get_url(name): return response.headers['x-amz-website-redirect-location'] def clone_images(): - DOWNLOAD_MATCH = r'nixos-\d\d.\d\d/latest-nixos-\w+-\w+-linux.\w+(.sha256)?' + DOWNLOAD_MATCH = r'nixos-(\d\d.\d\d|unstable)/latest-nixos-\w+-\w+-linux.\w+(.sha256)?' object_names = [ x.object_name diff --git a/proxmox.sh b/proxmox.sh index 22dbfdb..188f972 100755 --- a/proxmox.sh +++ b/proxmox.sh @@ -10,10 +10,16 @@ BASE_URL="${TUNASYNC_UPSTREAM_URL:-"http://download.proxmox.com"}" BASE_PATH="${TUNASYNC_WORKING_DIR}" APT_PATH="${BASE_PATH}/debian" +PBS_PATH="${BASE_PATH}/pbs" +PBS_CLIENT_PATH="${BASE_PATH}/pbs-client" +PMG_PATH="${BASE_PATH}/pmg" # === download deb packages ==== "$apt_sync" --delete "${BASE_URL}/debian" @debian-current pve-no-subscription,pvetest amd64 "$APT_PATH" +"$apt_sync" --delete "${BASE_URL}/debian/pbs" @debian-current pbs-no-subscription amd64 "$PBS_PATH" +"$apt_sync" --delete "${BASE_URL}/debian/pbs-client" @debian-current main amd64 "$PBS_CLIENT_PATH" +"$apt_sync" --delete "${BASE_URL}/debian/pmg" @debian-current pmg-no-subscription amd64 "$PMG_PATH" echo "Debian finished" # === download standalone files ==== diff --git a/pypi.sh b/pypi.sh index 36a934a..a703185 100755 --- a/pypi.sh +++ b/pypi.sh @@ -24,6 +24,7 @@ if [[ $INIT == "0" ]]; then cat << EOF [mirror] directory = ${TUNASYNC_WORKING_DIR} +storage-backend = filesystem master = ${PYPI_MASTER} ${DOWNLOAD_MIRROR} json = true diff --git a/s3.sh b/s3.sh index b7ea9ac..363c325 100755 --- a/s3.sh +++ b/s3.sh @@ -8,5 +8,5 @@ fi [[ ! -d "${TUNASYNC_WORKING_DIR}" ]] && mkdir -p "${TUNASYNC_WORKING_DIR}" mkdir /tmp/none; cd /tmp/none # enter an empty folder, so the stars in TUNASYNC_AWS_OPTIONS are not expanded -exec aws --no-sign-request ${ENDPOINT} s3 sync ${TUNASYNC_AWS_OPTIONS} "${TUNASYNC_UPSTREAM_URL}" "${TUNASYNC_WORKING_DIR}" +exec aws --no-sign-request ${ENDPOINT} s3 sync --exact-timestamps ${TUNASYNC_AWS_OPTIONS} "${TUNASYNC_UPSTREAM_URL}" "${TUNASYNC_WORKING_DIR}" diff --git a/stackage.py b/stackage.py index 8150867..86353b8 100755 --- a/stackage.py +++ b/stackage.py @@ -10,6 +10,8 @@ import shutil import subprocess import yaml +MIRROR_BASE_URL = os.getenv("MIRROR_BASE_URL", 'https://mirrors.tuna.tsinghua.edu.cn/stackage') +MIRROR_GITHUB_RELEASE_URL = os.getenv("MIRROR_GITHUB_RELEASE_URL", 'https://mirrors.tuna.tsinghua.edu.cn/github-release') class StackageSession(object): def __init__(self): @@ -38,7 +40,8 @@ class StackageSession(object): d = yaml.load( requests .get('https://raw.githubusercontent.com/commercialhaskell/stackage-content/master/stack/stack-setup-2.yaml') - .content + .content, + Loader=yaml.FullLoader ) for platform in d['ghc']: for ver in d['ghc'][platform]: @@ -48,27 +51,29 @@ class StackageSession(object): d['ghc'][platform][ver]['sha1'], ) d['ghc'][platform][ver]['url'] = ( - 'http://mirrors.tuna.tsinghua.edu.cn/stackage/ghc/{}' - .format(d['ghc'][platform][ver]['url'].split('/')[-1]) + '{}/ghc/{}' + .format(MIRROR_BASE_URL, d['ghc'][platform][ver]['url'].split('/')[-1]) ) if 'msys2' in d: if 'windows32' in d['msys2']: - print('windows32') d['msys2']['windows32']['url'] = d['msys2']['windows32']['url'].replace( 'https://github.com/fpco/stackage-content/releases/download/', - 'https://mirrors.tuna.tsinghua.edu.cn/github-release/commercialhaskell/stackage-content/msys2-') + f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/msys2-') if 'windows64' in d['msys2']: - print('windows64') d['msys2']['windows64']['url'] = d['msys2']['windows64']['url'].replace( 'https://github.com/commercialhaskell/stackage-content/releases/download/', - 'https://mirrors.tuna.tsinghua.edu.cn/github-release/commercialhaskell/stackage-content/') + f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/') if 'sevenzexe-info' in d: - d['sevenzexe-info']['url'] = 'https://mirrors.tuna.tsinghua.edu.cn/github-raw/fpco/minghc/master/bin/7z.exe' + d['sevenzexe-info']['url'] = d['sevenzexe-info']['url'].replace( + 'https://github.com/commercialhaskell/stackage-content/releases/download/', + f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/') if 'sevenzdll-info' in d: - d['sevenzdll-info']['url'] = 'https://mirrors.tuna.tsinghua.edu.cn/github-raw/fpco/minghc/master/bin/7z.dll' + d['sevenzdll-info']['url'] = d['sevenzdll-info']['url'].replace( + 'https://github.com/commercialhaskell/stackage-content/releases/download/', + f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/') for i in ['portable-git', 'stack', 'ghcjs']: del d[i] diff --git a/virtualbox.sh b/virtualbox.sh index 2a6b95f..10e4093 100755 --- a/virtualbox.sh +++ b/virtualbox.sh @@ -32,7 +32,7 @@ echo "Debian and ubuntu finished" timeout -s INT 30 wget ${WGET_OPTIONS:-} -q -O "/tmp/index.html" "${BASE_URL}/" timeout -s INT 30 wget ${WGET_OPTIONS:-} -q -O "${BASE_PATH}/LATEST.TXT" "${BASE_URL}/LATEST.TXT" -for((major=4;major<=6;major++));do +for((major=4;major<=7;major++));do LATEST_VERSION=$(grep -P -o "\"$major\.[\\d\\.]+/\"" -r /tmp/index.html|tail -n 1) LATEST_VERSION=${LATEST_VERSION%/\"} LATEST_VERSION=${LATEST_VERSION#\"} @@ -89,10 +89,10 @@ LATEST_VERSION=`cat "${BASE_PATH}/LATEST.TXT"` for filename in ${BASE_PATH}/${LATEST_VERSION}/*.*; do case $filename in *Win.exe) - ln -sf "${filename}" "${BASE_PATH}/virtualbox-Win-latest.exe" + ln -sf "${filename#"$BASE_PATH/"}" "${BASE_PATH}/virtualbox-Win-latest.exe" ;; *OSX.dmg) - ln -sf "${filename}" "${BASE_PATH}/virtualbox-osx-latest.dmg" + ln -sf "${filename#"$BASE_PATH/"}" "${BASE_PATH}/virtualbox-osx-latest.dmg" ;; esac done diff --git a/wine-builds.sh b/wine-builds.sh index efbaf02..6f21638 100755 --- a/wine-builds.sh +++ b/wine-builds.sh @@ -10,10 +10,10 @@ BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://dl.winehq.org/wine-builds"} export REPO_SIZE_FILE=/tmp/reposize.$RANDOM -"$apt_sync" --delete "$BASE_URL/ubuntu" @ubuntu-lts main amd64,i386 "$BASE_PATH/ubuntu" +"$apt_sync" --delete "$BASE_URL/ubuntu" @ubuntu-lts main amd64,i386,all "$BASE_PATH/ubuntu" echo "APT for Ubuntu finished" -"$apt_sync" --delete "$BASE_URL/debian" @debian-latest2 main amd64,i386 "$BASE_PATH/debian" +"$apt_sync" --delete "$BASE_URL/debian" @debian-latest2 main amd64,i386,all "$BASE_PATH/debian" echo "APT for Debian finished" echo "APT finished"