yacloud/tunasync-scripts
1
0
Fork 0
mirror of https://github.com/tuna/tunasync-scripts.git synced 2025-04-19 11:42:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'tuna:master' into adoptium-rhel

Browse Source
This commit is contained in:
Thomas Lee 2022-11-07 20:44:25 +08:00 committed by GitHub
commit 86a396f849
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 218 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
anaconda.py
View File

@ -64,7 +64,7 @@ CONDA_CLOUD_REPOS = (
"c4aarch64/linux-aarch64", "c4aarch64/noarch",
"pytorch3d/linux-64", "pytorch3d/noarch",
"idaholab/linux-64", "idaholab/noarch",
"MindSpore/linux-64", "MindSpore/linux-aarch64", "MindSpore/osx-arm64", "MindSpore/osx-64", "MindSpore/win-64",
"MindSpore/linux-64", "MindSpore/linux-aarch64", "MindSpore/osx-arm64", "MindSpore/osx-64", "MindSpore/win-64", "MindSpore/noarch",
)
EXCLUDED_PACKAGES = (
@ -96,14 +96,26 @@ def md5_check(file: Path, md5: str = None):
m.update(buf)
return m.hexdigest() == md5
def sha256_check(file: Path, sha256: str = None):
m = hashlib.sha256()
with file.open('rb') as f:
while True:
buf = f.read(1*1024*1024)
if not buf:
break
m.update(buf)
return m.hexdigest() == sha256
def curl_download(remote_url: str, dst_file: Path, md5: str = None):
def curl_download(remote_url: str, dst_file: Path, sha256: str = None, md5: str = None):
sp.check_call([
"curl", "-o", str(dst_file),
"-sL", "--remote-time", "--show-error",
"--fail", "--retry", "10", "--speed-time", "15",
"--speed-limit", "5000", remote_url,
])
if sha256 and (not sha256_check(dst_file, sha256)):
return "SHA256 mismatch"
if md5 and (not md5_check(dst_file, md5)):
return "MD5 mismatch"
@ -141,7 +153,14 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool):
if meta['name'] in EXCLUDED_PACKAGES:
continue
file_size, md5 = meta['size'], meta['md5']
file_size = meta['size']
# prefer sha256 over md5
sha256 = None
md5 = None
if 'sha256' in meta:
sha256 = meta['sha256']
elif 'md5' in meta:
md5 = meta['md5']
total_size += file_size
pkg_url = '/'.join([repo_url, filename])
@ -162,7 +181,7 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool):
for retry in range(3):
logging.info("Downloading {}".format(filename))
try:
err = curl_download(pkg_url, dst_file_wip, md5=md5)
err = curl_download(pkg_url, dst_file_wip, sha256=sha256, md5=md5)
if err is None:
dst_file_wip.rename(dst_file)
except sp.CalledProcessError:
@ -208,12 +227,12 @@ def sync_installer(repo_url, local_dir: Path):
if len(tds) != 4:
continue
fname = tds[0].find('a').text
md5 = tds[3].text
if md5 == '<directory>' or len(md5) != 32:
sha256 = tds[3].text
if sha256 == '<directory>' or len(sha256) != 64:
continue
yield (fname, md5)
yield (fname, sha256)
for filename, md5 in remote_list():
for filename, sha256 in remote_list():
pkg_url = "/".join([repo_url, filename])
dst_file = local_dir / filename
dst_file_wip = local_dir / ('.downloading.' + filename)
@ -230,7 +249,7 @@ def sync_installer(repo_url, local_dir: Path):
# Do content verification on ~5% of files (see issue #25)
if (not len_avail or remote_filesize == local_filesize) and remote_date.timestamp() == local_mtime and \
(random.random() < 0.95 or md5_check(dst_file, md5)):
(random.random() < 0.95 or sha256_check(dst_file, sha256)):
logging.info("Skipping {}".format(filename))
# Stop the scanning if the most recent version is present
@ -247,7 +266,7 @@ def sync_installer(repo_url, local_dir: Path):
logging.info("Downloading {}".format(filename))
err = ''
try:
err = curl_download(pkg_url, dst_file_wip, md5=md5)
err = curl_download(pkg_url, dst_file_wip, sha256=sha256)
if err is None:
dst_file_wip.rename(dst_file)
except sp.CalledProcessError:

157
git-recursive.sh Executable file
View File

@ -0,0 +1,157 @@
#!/bin/bash
if [[ -z "$TUNASYNC_UPSTREAM_URL" ]];then
echo "Please set the TUNASYNC_UPSTREAM_URL"
exit 1
fi
if [[ ! -z "$RECURSIVE" ]];then
echo "Sync in a recursive mode"
fi
TMPDIR=${TMPDIR:-"/tmp"}
MIRROR_BASE_URL=${MIRROR_BASE_URL:-"https://mirrors.tuna.tsinghua.edu.cn/git/"}
WORKING_DIR_BASE=${WORKING_DIR_BASE:-"/data/mirrors/git/"}
GENERATED_SCRIPT=${GENERATED_SCRIPT:-"/data/mirrors/git/qemu/qemu.sh"}
if [[ ! -z "$RECURSIVE" ]]; then
echo "#!/usr/bin/env bash" > $GENERATED_SCRIPT.tmp
fi
function script_append() {
if [[ ! -z "$RECURSIVE" ]]; then
echo "$1" >> $GENERATED_SCRIPT.tmp
fi
}
depth=0
function echon() {
echo depth $depth $@
}
function repo_init() {
local upstream=$1
local working_dir=$2
git clone --mirror "$upstream" "$working_dir"
}
function update_linux_git() {
local upstream=$1
local working_dir=$2
cd "$working_dir"
echon "==== SYNC $upstream START ===="
git remote set-url origin "$upstream"
/usr/bin/timeout -s INT 3600 git remote -v update -p
local ret=$?
[[ $ret -ne 0 ]] && echon "git update failed with rc=$ret"
local head=$(git remote show origin | awk '/HEAD branch:/ {print $NF}')
[[ -n "$head" ]] && echo "ref: refs/heads/$head" > HEAD
objs=$(find objects -type f | wc -l)
[[ "$objs" -gt 8 ]] && git repack -a -b -d
sz=$(git count-objects -v|grep -Po '(?<=size-pack: )\d+')
total_size=$(($total_size+1024*$sz))
echon "==== SYNC $upstream DONE ===="
return $ret
}
function git_sync() {
local upstream=$1
local working_dir=$2
if [[ ! -f "$working_dir/HEAD" ]]; then
echon "Initializing $upstream mirror"
repo_init $1 $2
return $?
fi
update_linux_git $1 $2
}
function checkout_repo() {
local repo_dir="$1"
local work_tree="$2"
local commit="$3"
echon "Checkout $repo_dir to $work_tree"
rm -rf $work_tree
git clone "$repo_dir" "$work_tree"
local ret=$?
if [[ $ret -ne 0 ]]; then
echon "git clone failed with rc=$ret"
return $ret
fi
if [[ ! -z "$commit" ]]; then
echon "Worktree $work_tree switch to commit $commit"
git -C $work_tree checkout $commit
fi
local repo_dir_no_git=${repo_dir%%.git}
local gitmodules="$work_tree/.gitmodules"
if [[ -f "$gitmodules" ]]; then
local paths_str=$(cat $gitmodules | grep path | cut -d '=' -f 2 | sed 's/^[[:blank:]]*//')
local urls_str=$(cat $gitmodules | grep url | cut -d '=' -f 2 | sed 's/^[[:blank:]]*//')
local -a paths
local -a urls
readarray -t paths <<<"$paths_str"
readarray -t urls <<<"$urls_str"
local -i i
for ((i=0;i<${#paths[@]};i++)); do
local path=${paths[$i]}
# ignore empty .gitmodules
if [[ "$path" == "" ]]; then
continue
fi
local commit=$(git -C $work_tree submodule status $path | cut -d ' ' -f 1 | cut -d '-' -f 2)
local git_path=$repo_dir_no_git/$path.git
mkdir -p $git_path
local mirror_url=$(echo $git_path | sed "s#$WORKING_DIR_BASE#$MIRROR_BASE_URL#")
script_append "cat >>.git/config <<EOF"
script_append "[submodule \"$path\"]"
script_append " active = true"
script_append " url = $mirror_url"
script_append "EOF"
script_append "mkdir -p $path"
script_append "git clone $mirror_url $path"
script_append "git submodule init $path"
script_append "git submodule update $path"
script_append "pushd $path"
git_sync_recursive ${urls[$i]} $git_path $commit
script_append "popd"
done
fi
}
function git_sync_recursive() {
depth=$(($depth+1))
local upstream=$1
local working_dir=$2
# it is ok that commit=""
local commit=$3
git_sync $1 $2
local ret=$?
if [[ $ret -ne 0 ]]; then
echon "git sync failed with rc=$ret"
return $ret
fi
if [[ ! -z "$RECURSIVE" ]]; then
working_dir_name=$(basename $2)
working_dir_name_no_git=${working_dir_name%%.git}
checkout_repo $working_dir $TMPDIR/$working_dir_name_no_git $commit
fi
depth=$(($depth-1))
}
path=$(basename $TUNASYNC_WORKING_DIR)
path_no_git=${path%%.git}
mirror_url=$(echo $TUNASYNC_WORKING_DIR | sed "s#$WORKING_DIR_BASE#$MIRROR_BASE_URL#")
script_append "mkdir -p $path_no_git"
script_append "git clone $mirror_url $path_no_git"
script_append "pushd $path_no_git"
git_sync_recursive $TUNASYNC_UPSTREAM_URL $TUNASYNC_WORKING_DIR
script_append "git submodule absorbgitdirs"
script_append "popd"
if [[ ! -z "$RECURSIVE" ]]; then
mv $GENERATED_SCRIPT.tmp $GENERATED_SCRIPT
fi
echo "Total size is" $(numfmt --to=iec $total_size)

2
github-raw.py
View File

@ -28,8 +28,6 @@ def delete_line_with_gbpdistro(s: str) -> str:
REPOS = [
# owner/repo, tree, tree, tree, blob
## for stackage
["fpco/minghc", "master", "bin", "7z.exe"],
["fpco/minghc", "master", "bin", "7z.dll"],
["fpco/stackage-content", "master", "stack", "global-hints.yaml"],
## for rosdep
{ "path": ["ros/rosdistro", "master", "rosdep", "sources.list.d", "20-default.list"], "filter": [ raw_to_mirror, delete_line_with_gbpdistro ] },

3
github-release.py
View File

@ -15,6 +15,7 @@ BASE_URL = os.getenv("TUNASYNC_UPSTREAM_URL", "https://api.github.com/repos/")
WORKING_DIR = os.getenv("TUNASYNC_WORKING_DIR")
REPOS = [
"Homebrew/homebrew-portable-ruby", # Used by homebrew-bottles
"Homebrew/glibc-bootstrap", # Used by homebrew-bottles, see #1586
{"repo": "googlefonts/noto-fonts", "tarball": True},
{"repo": "googlefonts/noto-cjk", "tarball": True},
{"repo": "googlefonts/noto-emoji", "tarball": True},
@ -41,7 +42,7 @@ REPOS = [
"conda-forge/miniforge",
"texstudio-org/texstudio",
"Stellarium/stellarium",
"UNIDY2002/THUInfo",
"thu-info-community/thu-info-app",
{"repo": "obsproject/obs-studio", "versions": 5, "pre_release": False},
"com-lihaoyi/mill", # better scala build tool,
"dbeaver/dbeaver", # issue #1348

2
kde-neon.sh
View File

@ -12,7 +12,7 @@ APT_PATH="${BASE_PATH}/user"
export REPO_SIZE_FILE=/tmp/reposize.$RANDOM
# =================== APT repos ===============================
"$apt_sync" --delete "${BASE_URL}/user" focal,bionic main dep11,cnf,all,amd64,i386 "${BASE_PATH}/user"
"$apt_sync" --delete "${BASE_URL}/user" jammy,focal,bionic main dep11,cnf,all,amd64,i386 "${BASE_PATH}/user"
echo "APT finished"
"${_here}/helpers/size-sum.sh" $REPO_SIZE_FILE --rm

2
mongodb.sh
View File

@ -8,7 +8,7 @@ yum_sync="${_here}/yum-sync.py"
BASE_PATH="${TUNASYNC_WORKING_DIR}"
BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://repo.mongodb.org"}
MONGO_VERSIONS=("5.0" "4.4" "4.2")
MONGO_VERSIONS=("6.0" "5.0" "4.4" "4.2")
STABLE_VERSION="5.0"
YUM_PATH="${BASE_PATH}/yum"

2
nixos-images.py
View File

@ -99,7 +99,7 @@ def get_url(name):
return response.headers['x-amz-website-redirect-location']
def clone_images():
DOWNLOAD_MATCH = r'nixos-\d\d.\d\d/latest-nixos-\w+-\w+-linux.\w+(.sha256)?'
DOWNLOAD_MATCH = r'nixos-(\d\d.\d\d|unstable)/latest-nixos-\w+-\w+-linux.\w+(.sha256)?'
object_names = [
x.object_name

6
proxmox.sh
View File

@ -10,10 +10,16 @@ BASE_URL="${TUNASYNC_UPSTREAM_URL:-"http://download.proxmox.com"}"
BASE_PATH="${TUNASYNC_WORKING_DIR}"
APT_PATH="${BASE_PATH}/debian"
PBS_PATH="${BASE_PATH}/pbs"
PBS_CLIENT_PATH="${BASE_PATH}/pbs-client"
PMG_PATH="${BASE_PATH}/pmg"
# === download deb packages ====
"$apt_sync" --delete "${BASE_URL}/debian" @debian-current pve-no-subscription,pvetest amd64 "$APT_PATH"
"$apt_sync" --delete "${BASE_URL}/debian/pbs" @debian-current pbs-no-subscription amd64 "$PBS_PATH"
"$apt_sync" --delete "${BASE_URL}/debian/pbs-client" @debian-current main amd64 "$PBS_CLIENT_PATH"
"$apt_sync" --delete "${BASE_URL}/debian/pmg" @debian-current pmg-no-subscription amd64 "$PMG_PATH"
echo "Debian finished"
# === download standalone files ====

1
pypi.sh
View File

@ -24,6 +24,7 @@ if [[ $INIT == "0" ]]; then
cat << EOF
[mirror]
directory = ${TUNASYNC_WORKING_DIR}
storage-backend = filesystem
master = ${PYPI_MASTER}
${DOWNLOAD_MIRROR}
json = true

2
s3.sh
View File

@ -8,5 +8,5 @@ fi
[[ ! -d "${TUNASYNC_WORKING_DIR}" ]] && mkdir -p "${TUNASYNC_WORKING_DIR}"
mkdir /tmp/none; cd /tmp/none # enter an empty folder, so the stars in TUNASYNC_AWS_OPTIONS are not expanded
exec aws --no-sign-request ${ENDPOINT} s3 sync ${TUNASYNC_AWS_OPTIONS} "${TUNASYNC_UPSTREAM_URL}" "${TUNASYNC_WORKING_DIR}"
exec aws --no-sign-request ${ENDPOINT} s3 sync --exact-timestamps ${TUNASYNC_AWS_OPTIONS} "${TUNASYNC_UPSTREAM_URL}" "${TUNASYNC_WORKING_DIR}"

23
stackage.py
View File

@ -10,6 +10,8 @@ import shutil
import subprocess
import yaml
MIRROR_BASE_URL = os.getenv("MIRROR_BASE_URL", 'https://mirrors.tuna.tsinghua.edu.cn/stackage')
MIRROR_GITHUB_RELEASE_URL = os.getenv("MIRROR_GITHUB_RELEASE_URL", 'https://mirrors.tuna.tsinghua.edu.cn/github-release')
class StackageSession(object):
def __init__(self):
@ -38,7 +40,8 @@ class StackageSession(object):
d = yaml.load(
requests
.get('https://raw.githubusercontent.com/commercialhaskell/stackage-content/master/stack/stack-setup-2.yaml')
.content
.content,
Loader=yaml.FullLoader
)
for platform in d['ghc']:
for ver in d['ghc'][platform]:
@ -48,27 +51,29 @@ class StackageSession(object):
d['ghc'][platform][ver]['sha1'],
)
d['ghc'][platform][ver]['url'] = (
'http://mirrors.tuna.tsinghua.edu.cn/stackage/ghc/{}'
.format(d['ghc'][platform][ver]['url'].split('/')[-1])
'{}/ghc/{}'
.format(MIRROR_BASE_URL, d['ghc'][platform][ver]['url'].split('/')[-1])
)
if 'msys2' in d:
if 'windows32' in d['msys2']:
print('windows32')
d['msys2']['windows32']['url'] = d['msys2']['windows32']['url'].replace(
'https://github.com/fpco/stackage-content/releases/download/',
'https://mirrors.tuna.tsinghua.edu.cn/github-release/commercialhaskell/stackage-content/msys2-')
f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/msys2-')
if 'windows64' in d['msys2']:
print('windows64')
d['msys2']['windows64']['url'] = d['msys2']['windows64']['url'].replace(
'https://github.com/commercialhaskell/stackage-content/releases/download/',
'https://mirrors.tuna.tsinghua.edu.cn/github-release/commercialhaskell/stackage-content/')
f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/')
if 'sevenzexe-info' in d:
d['sevenzexe-info']['url'] = 'https://mirrors.tuna.tsinghua.edu.cn/github-raw/fpco/minghc/master/bin/7z.exe'
d['sevenzexe-info']['url'] = d['sevenzexe-info']['url'].replace(
'https://github.com/commercialhaskell/stackage-content/releases/download/',
f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/')
if 'sevenzdll-info' in d:
d['sevenzdll-info']['url'] = 'https://mirrors.tuna.tsinghua.edu.cn/github-raw/fpco/minghc/master/bin/7z.dll'
d['sevenzdll-info']['url'] = d['sevenzdll-info']['url'].replace(
'https://github.com/commercialhaskell/stackage-content/releases/download/',
f'{MIRROR_GITHUB_RELEASE_URL}/commercialhaskell/stackage-content/')
for i in ['portable-git', 'stack', 'ghcjs']:
del d[i]

6
virtualbox.sh
View File

@ -32,7 +32,7 @@ echo "Debian and ubuntu finished"
timeout -s INT 30 wget ${WGET_OPTIONS:-} -q -O "/tmp/index.html" "${BASE_URL}/"
timeout -s INT 30 wget ${WGET_OPTIONS:-} -q -O "${BASE_PATH}/LATEST.TXT" "${BASE_URL}/LATEST.TXT"
for((major=4;major<=6;major++));do
for((major=4;major<=7;major++));do
LATEST_VERSION=$(grep -P -o "\"$major\.[\\d\\.]+/\"" -r /tmp/index.html|tail -n 1)
LATEST_VERSION=${LATEST_VERSION%/\"}
LATEST_VERSION=${LATEST_VERSION#\"}
@ -89,10 +89,10 @@ LATEST_VERSION=`cat "${BASE_PATH}/LATEST.TXT"`
for filename in ${BASE_PATH}/${LATEST_VERSION}/*.*; do
case $filename in
*Win.exe)
ln -sf "${filename}" "${BASE_PATH}/virtualbox-Win-latest.exe"
ln -sf "${filename#"$BASE_PATH/"}" "${BASE_PATH}/virtualbox-Win-latest.exe"
;;
*OSX.dmg)
ln -sf "${filename}" "${BASE_PATH}/virtualbox-osx-latest.dmg"
ln -sf "${filename#"$BASE_PATH/"}" "${BASE_PATH}/virtualbox-osx-latest.dmg"
;;
esac
done

4
wine-builds.sh
View File

@ -10,10 +10,10 @@ BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://dl.winehq.org/wine-builds"}
export REPO_SIZE_FILE=/tmp/reposize.$RANDOM
"$apt_sync" --delete "$BASE_URL/ubuntu" @ubuntu-lts main amd64,i386 "$BASE_PATH/ubuntu"
"$apt_sync" --delete "$BASE_URL/ubuntu" @ubuntu-lts main amd64,i386,all "$BASE_PATH/ubuntu"
echo "APT for Ubuntu finished"
"$apt_sync" --delete "$BASE_URL/debian" @debian-latest2 main amd64,i386 "$BASE_PATH/debian"
"$apt_sync" --delete "$BASE_URL/debian" @debian-latest2 main amd64,i386,all "$BASE_PATH/debian"
echo "APT for Debian finished"
echo "APT finished"