From d128b223dca19db7a8669c5b6f235df6c63e9086 Mon Sep 17 00:00:00 2001 From: z4yx Date: Thu, 23 Apr 2020 13:32:03 +0800 Subject: [PATCH] migrate to API v3 (solve #68) --- adoptopenjdk.py | 129 ++++++++++++++++++++++++++++++++++++++++++++++++ adoptopenjdk.sh | 116 ------------------------------------------- 2 files changed, 129 insertions(+), 116 deletions(-) create mode 100755 adoptopenjdk.py delete mode 100755 adoptopenjdk.sh diff --git a/adoptopenjdk.py b/adoptopenjdk.py new file mode 100755 index 0000000..c8f2dff --- /dev/null +++ b/adoptopenjdk.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +import hashlib +import traceback +import json +import os +import re +import shutil +import subprocess as sp +import tempfile +import argparse +import time +from email.utils import parsedate_to_datetime +from pathlib import Path +from typing import List, Set, Tuple, IO +import requests + +DOWNLOAD_TIMEOUT = int(os.getenv('DOWNLOAD_TIMEOUT', '1800')) +BASE_PATH = os.getenv('TUNASYNC_WORKING_DIR') +BASE_URL = os.getenv('TUNASYNC_UPSTREAM_URL', "http://adoptopenjdk.jfrog.io/adoptopenjdk") +FEATURE_VERSIONS = range(8, 15) + +def download_file(url: str, dst_file: Path)->bool: + try: + start = time.time() + with requests.get(url, stream=True, timeout=(5, 10)) as r: + r.raise_for_status() + if 'last-modified' in r.headers: + remote_ts = parsedate_to_datetime( + r.headers['last-modified']).timestamp() + else: remote_ts = None + + with dst_file.open('wb') as f: + for chunk in r.iter_content(chunk_size=1024**2): + if time.time() - start > DOWNLOAD_TIMEOUT: + raise TimeoutError("Download timeout") + if chunk: # filter out keep-alive new chunks + f.write(chunk) + if remote_ts is not None: + os.utime(dst_file, (remote_ts, remote_ts)) + return True + except BaseException as e: + print(e, flush=True) + if dst_file.is_file(): + dst_file.unlink() + return False + +def check_file(dest_filename: Path, pkg_checksum: str, size: int)->bool: + if dest_filename.stat().st_size != size: + print(f"Wrong size of {dest_filename}, expected {size}") + return False + sha = hashlib.sha256() + with dest_filename.open("rb") as f: + for block in iter(lambda: f.read(1024**2), b""): + sha.update(block) + if sha.hexdigest() != pkg_checksum: + print(f"Invalid checksum of {dest_filename}, expected {pkg_checksum}") + return False + return True + +def download_release(ver: int, jvm_impl: str): + r = requests.get(f"https://api.adoptopenjdk.net/v3/assets/latest/{ver}/{jvm_impl}", timeout=(5, 10)) + r.raise_for_status() + rel_list = r.json() + rel_path = Path(BASE_PATH) / str(ver) + alive_files = set() + for rel in rel_list: + binary = rel['binary'] + if binary['image_type'] not in ('jre', 'jdk'): continue + dst_dir = rel_path / binary['image_type'] / binary['architecture'] / binary['os'] + dst_dir.mkdir(parents=True, exist_ok=True) + for f in ('package', 'installer'): + if f not in binary: continue + meta = binary[f] + filename, tmpfile = dst_dir / meta['name'], dst_dir / ('.' + meta['name']) + alive_files.add(str(filename.relative_to(rel_path))) + if filename.is_file() and filename.stat().st_size == meta['size']: + print(f"Skiping {filename}") + continue + + print(f"Downloading {tmpfile}", flush=True) + for retry in range(3): + if download_file(meta['link'], tmpfile) and \ + check_file(tmpfile, meta['checksum'], meta['size']): + tmpfile.rename(filename) + break + else: + print(f"Failed to download {meta['link']}", flush=True) + + on_disk = set([ + str(i.relative_to(rel_path)) for i in rel_path.glob('**/*.*')]) + deleting = on_disk - alive_files + # print(on_disk) + # print(alive_files) + print(f"Deleting {len(deleting)} old files", flush=True) + for i in deleting: + print("Deleting", i) + (rel_path/i).unlink() + +if __name__ == "__main__": + here = Path(os.path.abspath(__file__)).parent + # =================== standalone ========================== + for v in FEATURE_VERSIONS: + for jvm in ('hotspot', 'openj9'): + download_release(v, jvm) + # =================== APT repos ========================== + # "$apt_sync" --delete "${BASE_URL}/deb" @ubuntu-lts,@debian-current main amd64,armhf,arm64 "$BASE_PATH/deb" + sp.run([str(here/"apt-sync.py"), + '--delete', + f'{BASE_URL}/deb', + '@ubuntu-lts,@debian-current', + 'main', + 'amd64,armhf,arm64', + f"{BASE_PATH}/deb" + ], + check=True) + print("APT finished", flush=True) + # =================== YUM repos ========================== + # "$yum_sync" "${BASE_URL}/rpm/centos/@{os_ver}/@{arch}" 7-8 AdoptOpenJDK x86_64,aarch64 "centos@{os_ver}-@{arch}" "$BASE_PATH/rpm" + sp.run([str(here/"yum-sync.py"), + BASE_URL+'/rpm/centos/@{os_ver}/@{arch}', + '7-8', + 'AdoptOpenJDK', + 'x86_64,aarch64', + "centos@{os_ver}-@{arch}", + f"{BASE_PATH}/rpm" + ], + check=True) + print("YUM finished", flush=True) + diff --git a/adoptopenjdk.sh b/adoptopenjdk.sh deleted file mode 100755 index ecda0a9..0000000 --- a/adoptopenjdk.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -# requires: curl, sha256sum, awk, jq -set -e - -_here=`dirname $(realpath $0)` -apt_sync="${_here}/apt-sync.py" -yum_sync="${_here}/yum-sync.py" - -BASE_PATH="${TUNASYNC_WORKING_DIR}" -BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://adoptopenjdk.jfrog.io/adoptopenjdk"} - -# =================== APT repos =============================== -"$apt_sync" --delete "${BASE_URL}/deb" @ubuntu-lts,@debian-current main amd64,armhf,arm64 "$BASE_PATH/deb" -echo "APT finished" - -# =================== YUM repos ========================== -"$yum_sync" "${BASE_URL}/rpm/centos/@{os_ver}/@{arch}" 7-8 AdoptOpenJDK x86_64,aarch64 "centos@{os_ver}-@{arch}" "$BASE_PATH/rpm" -echo "YUM finished" - -# =================== standalone ========================== -# 参数为版本，比如8,11等 -function downloadRelease() { - remote_filelist="$BASE_PATH/$1/filelist" - mkdir -p "$BASE_PATH/$1" - echo -n "" >$remote_filelist - curl -s "https://api.adoptopenjdk.net/v2/latestAssets/releases/openjdk$1" | \ - jq -r '.[]| [.version,.binary_type,.architecture,.os,.binary_name,.binary_link,.checksum_link,.installer_name,.installer_link,.installer_checksum_link]| @tsv' | \ - while IFS=$'\t' read -r version binary_type architecture os binary_name binary_link checksum_link installer_name installer_link installer_checksum_link; do - mkdir -p "$BASE_PATH/$version/$binary_type/$architecture/$os/" || true - dest_filename="$BASE_PATH/$version/$binary_type/$architecture/$os/$binary_name" - echo "$dest_filename" >>$remote_filelist - echo "$dest_filename.sha256.txt" >>$remote_filelist - declare downloaded=false - if [[ -f $dest_filename ]]; then - echo "Skiping $binary_name" - downloaded=true - fi - local retry=0 - while [[ $retry -lt 3 && $downloaded != true ]]; do - echo "Downloading ${dest_filename}" - link="$binary_link" - download_and_check && { - downloaded=true - } - ((retry+=1)) - done - if [[ ! -z "$installer_name" ]]; then - dest_filename="$BASE_PATH/$version/$binary_type/$architecture/$os/$installer_name" - echo "$dest_filename" >>$remote_filelist - echo "$dest_filename.sha256.txt" >>$remote_filelist - downloaded=false - if [[ -f $dest_filename ]]; then - echo "Skiping $installer_name" - downloaded=true - fi - retry=0 - while [[ $retry -lt 3 && $downloaded != true ]]; do - echo "Downloading ${dest_filename}" - link="$installer_link" - checksum_link="$installer_checksum_link" - download_and_check && { - downloaded=true - } - ((retry+=1)) - done - fi - done -} - -function clean_old_releases() { - declare version=$1 - declare remote_filelist="$BASE_PATH/$version/filelist" - declare local_filelist="/tmp/filelist.local" - [[ ! -f "$remote_filelist" ]] && return 0 - find "$BASE_PATH/$version" -type f > ${local_filelist} - comm <(sort $remote_filelist) <(sort $local_filelist) -13 | while read file; do - echo "deleting ${file}" - # rm "${file}" - done -} - -function download_and_check() { - rm "${dest_filename}" "${dest_filename}.sha256.txt" 2>/dev/null || true - rm "${dest_filename}.tmp" "${dest_filename}.sha256.txt.tmp" 2>/dev/null || true - curl -s -S --fail -L ${CURL_OPTIONS:-} \ - -o "${dest_filename}.tmp" \ - "$link" - curl -s -S --fail -L ${CURL_OPTIONS:-} \ - -o "${dest_filename}.sha256.txt.tmp" \ - "$checksum_link" || { - echo "Warning: ${dest_filename}.sha256.txt not exist, skipping SHA256 check" - mv "${dest_filename}.tmp" "${dest_filename}" - return 0 - } - sha256sum_check && { - mv "${dest_filename}.sha256.txt.tmp" "${dest_filename}.sha256.txt" - mv "${dest_filename}.tmp" "${dest_filename}" - return 0 - } -} - -function sha256sum_check() { - expected=$(cat "${dest_filename}.sha256.txt.tmp" | awk '{print $1}') - actual=$(sha256sum "${dest_filename}.tmp" | awk '{print $1}') - if [[ "$expected" = "$actual" ]]; then - return 0 - else - return 1 - fi -} - -for i in 8 9 10 11 12 13 14; -do - downloadRelease $i && clean_old_releases $i -done -