mirror of
https://github.com/tuna/tunasync-scripts.git
synced 2025-04-20 04:12:42 +00:00
migrate to API v3 (solve #68)
This commit is contained in:
z4yx
parent
d5d42685aa
commit
d128b223dc
129
adoptopenjdk.py
Executable file
129
adoptopenjdk.py
Executable file
@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import traceback
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess as sp
|
||||
import tempfile
|
||||
import argparse
|
||||
import time
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Set, Tuple, IO
|
||||
import requests
|
||||
|
||||
DOWNLOAD_TIMEOUT = int(os.getenv('DOWNLOAD_TIMEOUT', '1800'))
|
||||
BASE_PATH = os.getenv('TUNASYNC_WORKING_DIR')
|
||||
BASE_URL = os.getenv('TUNASYNC_UPSTREAM_URL', "http://adoptopenjdk.jfrog.io/adoptopenjdk")
|
||||
FEATURE_VERSIONS = range(8, 15)
|
||||
|
||||
def download_file(url: str, dst_file: Path)->bool:
|
||||
try:
|
||||
start = time.time()
|
||||
with requests.get(url, stream=True, timeout=(5, 10)) as r:
|
||||
r.raise_for_status()
|
||||
if 'last-modified' in r.headers:
|
||||
remote_ts = parsedate_to_datetime(
|
||||
r.headers['last-modified']).timestamp()
|
||||
else: remote_ts = None
|
||||
|
||||
with dst_file.open('wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=1024**2):
|
||||
if time.time() - start > DOWNLOAD_TIMEOUT:
|
||||
raise TimeoutError("Download timeout")
|
||||
if chunk: # filter out keep-alive new chunks
|
||||
f.write(chunk)
|
||||
if remote_ts is not None:
|
||||
os.utime(dst_file, (remote_ts, remote_ts))
|
||||
return True
|
||||
except BaseException as e:
|
||||
print(e, flush=True)
|
||||
if dst_file.is_file():
|
||||
dst_file.unlink()
|
||||
return False
|
||||
|
||||
def check_file(dest_filename: Path, pkg_checksum: str, size: int)->bool:
|
||||
if dest_filename.stat().st_size != size:
|
||||
print(f"Wrong size of {dest_filename}, expected {size}")
|
||||
return False
|
||||
sha = hashlib.sha256()
|
||||
with dest_filename.open("rb") as f:
|
||||
for block in iter(lambda: f.read(1024**2), b""):
|
||||
sha.update(block)
|
||||
if sha.hexdigest() != pkg_checksum:
|
||||
print(f"Invalid checksum of {dest_filename}, expected {pkg_checksum}")
|
||||
return False
|
||||
return True
|
||||
|
||||
def download_release(ver: int, jvm_impl: str):
|
||||
r = requests.get(f"https://api.adoptopenjdk.net/v3/assets/latest/{ver}/{jvm_impl}", timeout=(5, 10))
|
||||
r.raise_for_status()
|
||||
rel_list = r.json()
|
||||
rel_path = Path(BASE_PATH) / str(ver)
|
||||
alive_files = set()
|
||||
for rel in rel_list:
|
||||
binary = rel['binary']
|
||||
if binary['image_type'] not in ('jre', 'jdk'): continue
|
||||
dst_dir = rel_path / binary['image_type'] / binary['architecture'] / binary['os']
|
||||
dst_dir.mkdir(parents=True, exist_ok=True)
|
||||
for f in ('package', 'installer'):
|
||||
if f not in binary: continue
|
||||
meta = binary[f]
|
||||
filename, tmpfile = dst_dir / meta['name'], dst_dir / ('.' + meta['name'])
|
||||
alive_files.add(str(filename.relative_to(rel_path)))
|
||||
if filename.is_file() and filename.stat().st_size == meta['size']:
|
||||
print(f"Skiping {filename}")
|
||||
continue
|
||||
|
||||
print(f"Downloading {tmpfile}", flush=True)
|
||||
for retry in range(3):
|
||||
if download_file(meta['link'], tmpfile) and \
|
||||
check_file(tmpfile, meta['checksum'], meta['size']):
|
||||
tmpfile.rename(filename)
|
||||
break
|
||||
else:
|
||||
print(f"Failed to download {meta['link']}", flush=True)
|
||||
|
||||
on_disk = set([
|
||||
str(i.relative_to(rel_path)) for i in rel_path.glob('**/*.*')])
|
||||
deleting = on_disk - alive_files
|
||||
# print(on_disk)
|
||||
# print(alive_files)
|
||||
print(f"Deleting {len(deleting)} old files", flush=True)
|
||||
for i in deleting:
|
||||
print("Deleting", i)
|
||||
(rel_path/i).unlink()
|
||||
|
||||
if __name__ == "__main__":
|
||||
here = Path(os.path.abspath(__file__)).parent
|
||||
# =================== standalone ==========================
|
||||
for v in FEATURE_VERSIONS:
|
||||
for jvm in ('hotspot', 'openj9'):
|
||||
download_release(v, jvm)
|
||||
# =================== APT repos ==========================
|
||||
# "$apt_sync" --delete "${BASE_URL}/deb" @ubuntu-lts,@debian-current main amd64,armhf,arm64 "$BASE_PATH/deb"
|
||||
sp.run([str(here/"apt-sync.py"),
|
||||
'--delete',
|
||||
f'{BASE_URL}/deb',
|
||||
'@ubuntu-lts,@debian-current',
|
||||
'main',
|
||||
'amd64,armhf,arm64',
|
||||
f"{BASE_PATH}/deb"
|
||||
],
|
||||
check=True)
|
||||
print("APT finished", flush=True)
|
||||
# =================== YUM repos ==========================
|
||||
# "$yum_sync" "${BASE_URL}/rpm/centos/@{os_ver}/@{arch}" 7-8 AdoptOpenJDK x86_64,aarch64 "centos@{os_ver}-@{arch}" "$BASE_PATH/rpm"
|
||||
sp.run([str(here/"yum-sync.py"),
|
||||
BASE_URL+'/rpm/centos/@{os_ver}/@{arch}',
|
||||
'7-8',
|
||||
'AdoptOpenJDK',
|
||||
'x86_64,aarch64',
|
||||
"centos@{os_ver}-@{arch}",
|
||||
f"{BASE_PATH}/rpm"
|
||||
],
|
||||
check=True)
|
||||
print("YUM finished", flush=True)
|
||||
|
||||
116
adoptopenjdk.sh
116
adoptopenjdk.sh
@ -1,116 +0,0 @@
|
||||
#!/bin/bash
|
||||
# requires: curl, sha256sum, awk, jq
|
||||
set -e
|
||||
|
||||
_here=`dirname $(realpath $0)`
|
||||
apt_sync="${_here}/apt-sync.py"
|
||||
yum_sync="${_here}/yum-sync.py"
|
||||
|
||||
BASE_PATH="${TUNASYNC_WORKING_DIR}"
|
||||
BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://adoptopenjdk.jfrog.io/adoptopenjdk"}
|
||||
|
||||
# =================== APT repos ===============================
|
||||
"$apt_sync" --delete "${BASE_URL}/deb" @ubuntu-lts,@debian-current main amd64,armhf,arm64 "$BASE_PATH/deb"
|
||||
echo "APT finished"
|
||||
|
||||
# =================== YUM repos ==========================
|
||||
"$yum_sync" "${BASE_URL}/rpm/centos/@{os_ver}/@{arch}" 7-8 AdoptOpenJDK x86_64,aarch64 "centos@{os_ver}-@{arch}" "$BASE_PATH/rpm"
|
||||
echo "YUM finished"
|
||||
|
||||
# =================== standalone ==========================
|
||||
# 参数为版本，比如8,11等
|
||||
function downloadRelease() {
|
||||
remote_filelist="$BASE_PATH/$1/filelist"
|
||||
mkdir -p "$BASE_PATH/$1"
|
||||
echo -n "" >$remote_filelist
|
||||
curl -s "https://api.adoptopenjdk.net/v2/latestAssets/releases/openjdk$1" | \
|
||||
jq -r '.[]| [.version,.binary_type,.architecture,.os,.binary_name,.binary_link,.checksum_link,.installer_name,.installer_link,.installer_checksum_link]| @tsv' | \
|
||||
while IFS=$'\t' read -r version binary_type architecture os binary_name binary_link checksum_link installer_name installer_link installer_checksum_link; do
|
||||
mkdir -p "$BASE_PATH/$version/$binary_type/$architecture/$os/" || true
|
||||
dest_filename="$BASE_PATH/$version/$binary_type/$architecture/$os/$binary_name"
|
||||
echo "$dest_filename" >>$remote_filelist
|
||||
echo "$dest_filename.sha256.txt" >>$remote_filelist
|
||||
declare downloaded=false
|
||||
if [[ -f $dest_filename ]]; then
|
||||
echo "Skiping $binary_name"
|
||||
downloaded=true
|
||||
fi
|
||||
local retry=0
|
||||
while [[ $retry -lt 3 && $downloaded != true ]]; do
|
||||
echo "Downloading ${dest_filename}"
|
||||
link="$binary_link"
|
||||
download_and_check && {
|
||||
downloaded=true
|
||||
}
|
||||
((retry+=1))
|
||||
done
|
||||
if [[ ! -z "$installer_name" ]]; then
|
||||
dest_filename="$BASE_PATH/$version/$binary_type/$architecture/$os/$installer_name"
|
||||
echo "$dest_filename" >>$remote_filelist
|
||||
echo "$dest_filename.sha256.txt" >>$remote_filelist
|
||||
downloaded=false
|
||||
if [[ -f $dest_filename ]]; then
|
||||
echo "Skiping $installer_name"
|
||||
downloaded=true
|
||||
fi
|
||||
retry=0
|
||||
while [[ $retry -lt 3 && $downloaded != true ]]; do
|
||||
echo "Downloading ${dest_filename}"
|
||||
link="$installer_link"
|
||||
checksum_link="$installer_checksum_link"
|
||||
download_and_check && {
|
||||
downloaded=true
|
||||
}
|
||||
((retry+=1))
|
||||
done
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function clean_old_releases() {
|
||||
declare version=$1
|
||||
declare remote_filelist="$BASE_PATH/$version/filelist"
|
||||
declare local_filelist="/tmp/filelist.local"
|
||||
[[ ! -f "$remote_filelist" ]] && return 0
|
||||
find "$BASE_PATH/$version" -type f > ${local_filelist}
|
||||
comm <(sort $remote_filelist) <(sort $local_filelist) -13 | while read file; do
|
||||
echo "deleting ${file}"
|
||||
# rm "${file}"
|
||||
done
|
||||
}
|
||||
|
||||
function download_and_check() {
|
||||
rm "${dest_filename}" "${dest_filename}.sha256.txt" 2>/dev/null || true
|
||||
rm "${dest_filename}.tmp" "${dest_filename}.sha256.txt.tmp" 2>/dev/null || true
|
||||
curl -s -S --fail -L ${CURL_OPTIONS:-} \
|
||||
-o "${dest_filename}.tmp" \
|
||||
"$link"
|
||||
curl -s -S --fail -L ${CURL_OPTIONS:-} \
|
||||
-o "${dest_filename}.sha256.txt.tmp" \
|
||||
"$checksum_link" || {
|
||||
echo "Warning: ${dest_filename}.sha256.txt not exist, skipping SHA256 check"
|
||||
mv "${dest_filename}.tmp" "${dest_filename}"
|
||||
return 0
|
||||
}
|
||||
sha256sum_check && {
|
||||
mv "${dest_filename}.sha256.txt.tmp" "${dest_filename}.sha256.txt"
|
||||
mv "${dest_filename}.tmp" "${dest_filename}"
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
function sha256sum_check() {
|
||||
expected=$(cat "${dest_filename}.sha256.txt.tmp" | awk '{print $1}')
|
||||
actual=$(sha256sum "${dest_filename}.tmp" | awk '{print $1}')
|
||||
if [[ "$expected" = "$actual" ]]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
for i in 8 9 10 11 12 13 14;
|
||||
do
|
||||
downloadRelease $i && clean_old_releases $i
|
||||
done
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user