mirror of
https://github.com/tuna/tunasync-scripts.git
synced 2025-04-20 12:42:50 +00:00
migrate to API v3 (solve #68)
This commit is contained in:
z4yx
parent
d5d42685aa
commit
d128b223dc
129
adoptopenjdk.py
Executable file
129
adoptopenjdk.py
Executable file
@ -0,0 +1,129 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import hashlib
|
||||||
|
import traceback
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess as sp
|
||||||
|
import tempfile
|
||||||
|
import argparse
|
||||||
|
import time
|
||||||
|
from email.utils import parsedate_to_datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, Set, Tuple, IO
|
||||||
|
import requests
|
||||||
|
|
||||||
|
DOWNLOAD_TIMEOUT = int(os.getenv('DOWNLOAD_TIMEOUT', '1800'))
|
||||||
|
BASE_PATH = os.getenv('TUNASYNC_WORKING_DIR')
|
||||||
|
BASE_URL = os.getenv('TUNASYNC_UPSTREAM_URL', "http://adoptopenjdk.jfrog.io/adoptopenjdk")
|
||||||
|
FEATURE_VERSIONS = range(8, 15)
|
||||||
|
|
||||||
|
def download_file(url: str, dst_file: Path)->bool:
|
||||||
|
try:
|
||||||
|
start = time.time()
|
||||||
|
with requests.get(url, stream=True, timeout=(5, 10)) as r:
|
||||||
|
r.raise_for_status()
|
||||||
|
if 'last-modified' in r.headers:
|
||||||
|
remote_ts = parsedate_to_datetime(
|
||||||
|
r.headers['last-modified']).timestamp()
|
||||||
|
else: remote_ts = None
|
||||||
|
|
||||||
|
with dst_file.open('wb') as f:
|
||||||
|
for chunk in r.iter_content(chunk_size=1024**2):
|
||||||
|
if time.time() - start > DOWNLOAD_TIMEOUT:
|
||||||
|
raise TimeoutError("Download timeout")
|
||||||
|
if chunk: # filter out keep-alive new chunks
|
||||||
|
f.write(chunk)
|
||||||
|
if remote_ts is not None:
|
||||||
|
os.utime(dst_file, (remote_ts, remote_ts))
|
||||||
|
return True
|
||||||
|
except BaseException as e:
|
||||||
|
print(e, flush=True)
|
||||||
|
if dst_file.is_file():
|
||||||
|
dst_file.unlink()
|
||||||
|
return False
|
||||||
|
|
||||||
|
def check_file(dest_filename: Path, pkg_checksum: str, size: int)->bool:
|
||||||
|
if dest_filename.stat().st_size != size:
|
||||||
|
print(f"Wrong size of {dest_filename}, expected {size}")
|
||||||
|
return False
|
||||||
|
sha = hashlib.sha256()
|
||||||
|
with dest_filename.open("rb") as f:
|
||||||
|
for block in iter(lambda: f.read(1024**2), b""):
|
||||||
|
sha.update(block)
|
||||||
|
if sha.hexdigest() != pkg_checksum:
|
||||||
|
print(f"Invalid checksum of {dest_filename}, expected {pkg_checksum}")
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def download_release(ver: int, jvm_impl: str):
|
||||||
|
r = requests.get(f"https://api.adoptopenjdk.net/v3/assets/latest/{ver}/{jvm_impl}", timeout=(5, 10))
|
||||||
|
r.raise_for_status()
|
||||||
|
rel_list = r.json()
|
||||||
|
rel_path = Path(BASE_PATH) / str(ver)
|
||||||
|
alive_files = set()
|
||||||
|
for rel in rel_list:
|
||||||
|
binary = rel['binary']
|
||||||
|
if binary['image_type'] not in ('jre', 'jdk'): continue
|
||||||
|
dst_dir = rel_path / binary['image_type'] / binary['architecture'] / binary['os']
|
||||||
|
dst_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
for f in ('package', 'installer'):
|
||||||
|
if f not in binary: continue
|
||||||
|
meta = binary[f]
|
||||||
|
filename, tmpfile = dst_dir / meta['name'], dst_dir / ('.' + meta['name'])
|
||||||
|
alive_files.add(str(filename.relative_to(rel_path)))
|
||||||
|
if filename.is_file() and filename.stat().st_size == meta['size']:
|
||||||
|
print(f"Skiping {filename}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
print(f"Downloading {tmpfile}", flush=True)
|
||||||
|
for retry in range(3):
|
||||||
|
if download_file(meta['link'], tmpfile) and \
|
||||||
|
check_file(tmpfile, meta['checksum'], meta['size']):
|
||||||
|
tmpfile.rename(filename)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
print(f"Failed to download {meta['link']}", flush=True)
|
||||||
|
|
||||||
|
on_disk = set([
|
||||||
|
str(i.relative_to(rel_path)) for i in rel_path.glob('**/*.*')])
|
||||||
|
deleting = on_disk - alive_files
|
||||||
|
# print(on_disk)
|
||||||
|
# print(alive_files)
|
||||||
|
print(f"Deleting {len(deleting)} old files", flush=True)
|
||||||
|
for i in deleting:
|
||||||
|
print("Deleting", i)
|
||||||
|
(rel_path/i).unlink()
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
here = Path(os.path.abspath(__file__)).parent
|
||||||
|
# =================== standalone ==========================
|
||||||
|
for v in FEATURE_VERSIONS:
|
||||||
|
for jvm in ('hotspot', 'openj9'):
|
||||||
|
download_release(v, jvm)
|
||||||
|
# =================== APT repos ==========================
|
||||||
|
# "$apt_sync" --delete "${BASE_URL}/deb" @ubuntu-lts,@debian-current main amd64,armhf,arm64 "$BASE_PATH/deb"
|
||||||
|
sp.run([str(here/"apt-sync.py"),
|
||||||
|
'--delete',
|
||||||
|
f'{BASE_URL}/deb',
|
||||||
|
'@ubuntu-lts,@debian-current',
|
||||||
|
'main',
|
||||||
|
'amd64,armhf,arm64',
|
||||||
|
f"{BASE_PATH}/deb"
|
||||||
|
],
|
||||||
|
check=True)
|
||||||
|
print("APT finished", flush=True)
|
||||||
|
# =================== YUM repos ==========================
|
||||||
|
# "$yum_sync" "${BASE_URL}/rpm/centos/@{os_ver}/@{arch}" 7-8 AdoptOpenJDK x86_64,aarch64 "centos@{os_ver}-@{arch}" "$BASE_PATH/rpm"
|
||||||
|
sp.run([str(here/"yum-sync.py"),
|
||||||
|
BASE_URL+'/rpm/centos/@{os_ver}/@{arch}',
|
||||||
|
'7-8',
|
||||||
|
'AdoptOpenJDK',
|
||||||
|
'x86_64,aarch64',
|
||||||
|
"centos@{os_ver}-@{arch}",
|
||||||
|
f"{BASE_PATH}/rpm"
|
||||||
|
],
|
||||||
|
check=True)
|
||||||
|
print("YUM finished", flush=True)
|
||||||
|
|
||||||
116
adoptopenjdk.sh
116
adoptopenjdk.sh
@ -1,116 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# requires: curl, sha256sum, awk, jq
|
|
||||||
set -e
|
|
||||||
|
|
||||||
_here=`dirname $(realpath $0)`
|
|
||||||
apt_sync="${_here}/apt-sync.py"
|
|
||||||
yum_sync="${_here}/yum-sync.py"
|
|
||||||
|
|
||||||
BASE_PATH="${TUNASYNC_WORKING_DIR}"
|
|
||||||
BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://adoptopenjdk.jfrog.io/adoptopenjdk"}
|
|
||||||
|
|
||||||
# =================== APT repos ===============================
|
|
||||||
"$apt_sync" --delete "${BASE_URL}/deb" @ubuntu-lts,@debian-current main amd64,armhf,arm64 "$BASE_PATH/deb"
|
|
||||||
echo "APT finished"
|
|
||||||
|
|
||||||
# =================== YUM repos ==========================
|
|
||||||
"$yum_sync" "${BASE_URL}/rpm/centos/@{os_ver}/@{arch}" 7-8 AdoptOpenJDK x86_64,aarch64 "centos@{os_ver}-@{arch}" "$BASE_PATH/rpm"
|
|
||||||
echo "YUM finished"
|
|
||||||
|
|
||||||
# =================== standalone ==========================
|
|
||||||
# 参数为版本，比如8,11等
|
|
||||||
function downloadRelease() {
|
|
||||||
remote_filelist="$BASE_PATH/$1/filelist"
|
|
||||||
mkdir -p "$BASE_PATH/$1"
|
|
||||||
echo -n "" >$remote_filelist
|
|
||||||
curl -s "https://api.adoptopenjdk.net/v2/latestAssets/releases/openjdk$1" | \
|
|
||||||
jq -r '.[]| [.version,.binary_type,.architecture,.os,.binary_name,.binary_link,.checksum_link,.installer_name,.installer_link,.installer_checksum_link]| @tsv' | \
|
|
||||||
while IFS=$'\t' read -r version binary_type architecture os binary_name binary_link checksum_link installer_name installer_link installer_checksum_link; do
|
|
||||||
mkdir -p "$BASE_PATH/$version/$binary_type/$architecture/$os/" || true
|
|
||||||
dest_filename="$BASE_PATH/$version/$binary_type/$architecture/$os/$binary_name"
|
|
||||||
echo "$dest_filename" >>$remote_filelist
|
|
||||||
echo "$dest_filename.sha256.txt" >>$remote_filelist
|
|
||||||
declare downloaded=false
|
|
||||||
if [[ -f $dest_filename ]]; then
|
|
||||||
echo "Skiping $binary_name"
|
|
||||||
downloaded=true
|
|
||||||
fi
|
|
||||||
local retry=0
|
|
||||||
while [[ $retry -lt 3 && $downloaded != true ]]; do
|
|
||||||
echo "Downloading ${dest_filename}"
|
|
||||||
link="$binary_link"
|
|
||||||
download_and_check && {
|
|
||||||
downloaded=true
|
|
||||||
}
|
|
||||||
((retry+=1))
|
|
||||||
done
|
|
||||||
if [[ ! -z "$installer_name" ]]; then
|
|
||||||
dest_filename="$BASE_PATH/$version/$binary_type/$architecture/$os/$installer_name"
|
|
||||||
echo "$dest_filename" >>$remote_filelist
|
|
||||||
echo "$dest_filename.sha256.txt" >>$remote_filelist
|
|
||||||
downloaded=false
|
|
||||||
if [[ -f $dest_filename ]]; then
|
|
||||||
echo "Skiping $installer_name"
|
|
||||||
downloaded=true
|
|
||||||
fi
|
|
||||||
retry=0
|
|
||||||
while [[ $retry -lt 3 && $downloaded != true ]]; do
|
|
||||||
echo "Downloading ${dest_filename}"
|
|
||||||
link="$installer_link"
|
|
||||||
checksum_link="$installer_checksum_link"
|
|
||||||
download_and_check && {
|
|
||||||
downloaded=true
|
|
||||||
}
|
|
||||||
((retry+=1))
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
function clean_old_releases() {
|
|
||||||
declare version=$1
|
|
||||||
declare remote_filelist="$BASE_PATH/$version/filelist"
|
|
||||||
declare local_filelist="/tmp/filelist.local"
|
|
||||||
[[ ! -f "$remote_filelist" ]] && return 0
|
|
||||||
find "$BASE_PATH/$version" -type f > ${local_filelist}
|
|
||||||
comm <(sort $remote_filelist) <(sort $local_filelist) -13 | while read file; do
|
|
||||||
echo "deleting ${file}"
|
|
||||||
# rm "${file}"
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
function download_and_check() {
|
|
||||||
rm "${dest_filename}" "${dest_filename}.sha256.txt" 2>/dev/null || true
|
|
||||||
rm "${dest_filename}.tmp" "${dest_filename}.sha256.txt.tmp" 2>/dev/null || true
|
|
||||||
curl -s -S --fail -L ${CURL_OPTIONS:-} \
|
|
||||||
-o "${dest_filename}.tmp" \
|
|
||||||
"$link"
|
|
||||||
curl -s -S --fail -L ${CURL_OPTIONS:-} \
|
|
||||||
-o "${dest_filename}.sha256.txt.tmp" \
|
|
||||||
"$checksum_link" || {
|
|
||||||
echo "Warning: ${dest_filename}.sha256.txt not exist, skipping SHA256 check"
|
|
||||||
mv "${dest_filename}.tmp" "${dest_filename}"
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
sha256sum_check && {
|
|
||||||
mv "${dest_filename}.sha256.txt.tmp" "${dest_filename}.sha256.txt"
|
|
||||||
mv "${dest_filename}.tmp" "${dest_filename}"
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function sha256sum_check() {
|
|
||||||
expected=$(cat "${dest_filename}.sha256.txt.tmp" | awk '{print $1}')
|
|
||||||
actual=$(sha256sum "${dest_filename}.tmp" | awk '{print $1}')
|
|
||||||
if [[ "$expected" = "$actual" ]]; then
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
for i in 8 9 10 11 12 13 14;
|
|
||||||
do
|
|
||||||
downloadRelease $i && clean_old_releases $i
|
|
||||||
done
|
|
||||||
|
|
||||||
Loading…
x
Reference in New Issue
Block a user