migrate to API v3 (solve #68)

adoptopenjdk.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+import hashlib
+import traceback
+import json
+import os
+import re
+import shutil
+import subprocess as sp
+import tempfile
+import argparse
+import time
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+from typing import List, Set, Tuple, IO
+import requests
+
+DOWNLOAD_TIMEOUT = int(os.getenv('DOWNLOAD_TIMEOUT', '1800'))
+BASE_PATH = os.getenv('TUNASYNC_WORKING_DIR')
+BASE_URL = os.getenv('TUNASYNC_UPSTREAM_URL', "http://adoptopenjdk.jfrog.io/adoptopenjdk")
+FEATURE_VERSIONS = range(8, 15)
+
+def download_file(url: str, dst_file: Path)->bool:
+    try:
+        start = time.time()
+        with requests.get(url, stream=True, timeout=(5, 10)) as r:
+            r.raise_for_status()
+            if 'last-modified' in r.headers:
+                remote_ts = parsedate_to_datetime(
+                    r.headers['last-modified']).timestamp()
+            else: remote_ts = None
+
+            with dst_file.open('wb') as f:
+                for chunk in r.iter_content(chunk_size=1024**2):
+                    if time.time() - start > DOWNLOAD_TIMEOUT:
+                        raise TimeoutError("Download timeout")
+                    if chunk:  # filter out keep-alive new chunks
+                        f.write(chunk)
+            if remote_ts is not None:
+                os.utime(dst_file, (remote_ts, remote_ts))
+        return True
+    except BaseException as e:
+        print(e, flush=True)
+        if dst_file.is_file():
+            dst_file.unlink()
+    return False
+
+def check_file(dest_filename: Path, pkg_checksum: str, size: int)->bool:
+    if dest_filename.stat().st_size != size:
+        print(f"Wrong size of {dest_filename}, expected {size}")
+        return False
+    sha = hashlib.sha256()
+    with dest_filename.open("rb") as f:
+        for block in iter(lambda: f.read(1024**2), b""):
+            sha.update(block)
+    if sha.hexdigest() != pkg_checksum:
+        print(f"Invalid checksum of {dest_filename}, expected {pkg_checksum}")
+        return False
+    return True
+
+def download_release(ver: int, jvm_impl: str):
+    r = requests.get(f"https://api.adoptopenjdk.net/v3/assets/latest/{ver}/{jvm_impl}", timeout=(5, 10))
+    r.raise_for_status()
+    rel_list = r.json()
+    rel_path = Path(BASE_PATH) / str(ver)
+    alive_files = set()
+    for rel in rel_list:
+        binary = rel['binary']
+        if binary['image_type'] not in ('jre', 'jdk'): continue
+        dst_dir = rel_path / binary['image_type'] / binary['architecture'] / binary['os']
+        dst_dir.mkdir(parents=True, exist_ok=True)
+        for f in ('package', 'installer'):
+            if f not in binary: continue
+            meta = binary[f]
+            filename, tmpfile = dst_dir / meta['name'], dst_dir / ('.' + meta['name'])
+            alive_files.add(str(filename.relative_to(rel_path)))
+            if filename.is_file() and filename.stat().st_size == meta['size']:
+                print(f"Skiping {filename}")
+                continue
+
+            print(f"Downloading {tmpfile}", flush=True)
+            for retry in range(3):
+                if download_file(meta['link'], tmpfile) and \
+                    check_file(tmpfile, meta['checksum'], meta['size']):
+                    tmpfile.rename(filename)
+                    break
+            else:
+                print(f"Failed to download {meta['link']}", flush=True)
+
+    on_disk = set([
+        str(i.relative_to(rel_path)) for i in rel_path.glob('**/*.*')])
+    deleting = on_disk - alive_files
+    # print(on_disk)
+    # print(alive_files)
+    print(f"Deleting {len(deleting)} old files", flush=True)
+    for i in deleting:
+        print("Deleting", i)
+        (rel_path/i).unlink()
+
+if __name__ == "__main__":
+    here = Path(os.path.abspath(__file__)).parent
+    # =================== standalone ==========================
+    for v in FEATURE_VERSIONS:
+        for jvm in ('hotspot', 'openj9'):
+            download_release(v, jvm)
+    # =================== APT repos ==========================
+    # "$apt_sync" --delete "${BASE_URL}/deb" @ubuntu-lts,@debian-current main amd64,armhf,arm64 "$BASE_PATH/deb"
+    sp.run([str(here/"apt-sync.py"),
+        '--delete',
+        f'{BASE_URL}/deb',
+        '@ubuntu-lts,@debian-current',
+        'main',
+        'amd64,armhf,arm64',
+        f"{BASE_PATH}/deb"
+        ],
+        check=True)
+    print("APT finished", flush=True)
+    # =================== YUM repos ==========================
+    # "$yum_sync" "${BASE_URL}/rpm/centos/@{os_ver}/@{arch}" 7-8 AdoptOpenJDK x86_64,aarch64 "centos@{os_ver}-@{arch}" "$BASE_PATH/rpm"
+    sp.run([str(here/"yum-sync.py"),
+        BASE_URL+'/rpm/centos/@{os_ver}/@{arch}',
+        '7-8',
+        'AdoptOpenJDK',
+        'x86_64,aarch64',
+        "centos@{os_ver}-@{arch}",
+        f"{BASE_PATH}/rpm"
+        ],
+        check=True)
+    print("YUM finished", flush=True)
+

adoptopenjdk.sh

@@ -1,116 +0,0 @@
-#!/bin/bash
-# requires: curl, sha256sum, awk, jq
-set -e
-
-_here=`dirname $(realpath $0)`
-apt_sync="${_here}/apt-sync.py" 
-yum_sync="${_here}/yum-sync.py"
-
-BASE_PATH="${TUNASYNC_WORKING_DIR}"
-BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://adoptopenjdk.jfrog.io/adoptopenjdk"}
-
-# =================== APT repos ===============================
-"$apt_sync" --delete "${BASE_URL}/deb" @ubuntu-lts,@debian-current main amd64,armhf,arm64 "$BASE_PATH/deb"
-echo "APT finished"
-
-# =================== YUM repos ==========================
-"$yum_sync" "${BASE_URL}/rpm/centos/@{os_ver}/@{arch}" 7-8 AdoptOpenJDK x86_64,aarch64 "centos@{os_ver}-@{arch}" "$BASE_PATH/rpm"
-echo "YUM finished"
-
-# =================== standalone ==========================
-# 参数为版本，比如8,11等
-function downloadRelease() {
-  remote_filelist="$BASE_PATH/$1/filelist"
-  mkdir -p "$BASE_PATH/$1"
-  echo -n "" >$remote_filelist
-  curl -s "https://api.adoptopenjdk.net/v2/latestAssets/releases/openjdk$1" | \
-    jq -r '.[]| [.version,.binary_type,.architecture,.os,.binary_name,.binary_link,.checksum_link,.installer_name,.installer_link,.installer_checksum_link]| @tsv' | \
-    while IFS=$'\t' read -r version binary_type architecture os binary_name binary_link checksum_link installer_name installer_link installer_checksum_link; do
-      mkdir -p "$BASE_PATH/$version/$binary_type/$architecture/$os/" || true
-      dest_filename="$BASE_PATH/$version/$binary_type/$architecture/$os/$binary_name"
-      echo "$dest_filename" >>$remote_filelist
-      echo "$dest_filename.sha256.txt" >>$remote_filelist
-      declare downloaded=false
-      if [[ -f $dest_filename ]]; then
-        echo "Skiping $binary_name"
-        downloaded=true
-      fi
-      local retry=0
-      while [[ $retry -lt 3 && $downloaded != true ]]; do
-        echo "Downloading ${dest_filename}"
-        link="$binary_link"
-        download_and_check && {
-            downloaded=true
-        }
-        ((retry+=1))
-      done
-      if [[ ! -z "$installer_name" ]]; then
-        dest_filename="$BASE_PATH/$version/$binary_type/$architecture/$os/$installer_name"
-        echo "$dest_filename" >>$remote_filelist
-        echo "$dest_filename.sha256.txt" >>$remote_filelist
-        downloaded=false
-        if [[ -f $dest_filename ]]; then
-          echo "Skiping $installer_name"
-          downloaded=true
-        fi
-        retry=0
-        while [[ $retry -lt 3 && $downloaded != true ]]; do
-          echo "Downloading ${dest_filename}"
-          link="$installer_link"
-          checksum_link="$installer_checksum_link"
-          download_and_check && {
-            downloaded=true
-          }
-          ((retry+=1))
-        done
-      fi
-    done
-}
-
-function clean_old_releases() {
-  declare version=$1
-  declare remote_filelist="$BASE_PATH/$version/filelist"
-  declare local_filelist="/tmp/filelist.local"
-  [[ ! -f "$remote_filelist" ]] && return 0
-  find "$BASE_PATH/$version" -type f > ${local_filelist}
-  comm <(sort $remote_filelist) <(sort $local_filelist) -13 | while read file; do
-      echo "deleting ${file}"
-      # rm "${file}"
-  done
-}
-
-function download_and_check() {
-  rm "${dest_filename}" "${dest_filename}.sha256.txt" 2>/dev/null || true
-  rm "${dest_filename}.tmp" "${dest_filename}.sha256.txt.tmp" 2>/dev/null || true
-  curl -s -S --fail -L ${CURL_OPTIONS:-}  \
-    -o "${dest_filename}.tmp" \
-    "$link"
-  curl -s -S --fail -L ${CURL_OPTIONS:-}  \
-    -o "${dest_filename}.sha256.txt.tmp" \
-    "$checksum_link" || {
-    echo "Warning: ${dest_filename}.sha256.txt not exist, skipping SHA256 check"
-    mv "${dest_filename}.tmp" "${dest_filename}"
-    return 0
-  }
-  sha256sum_check && {
-    mv "${dest_filename}.sha256.txt.tmp" "${dest_filename}.sha256.txt"
-    mv "${dest_filename}.tmp" "${dest_filename}"
-    return 0
-  }
-}
-
-function sha256sum_check() {
-  expected=$(cat "${dest_filename}.sha256.txt.tmp" | awk '{print $1}')
-  actual=$(sha256sum "${dest_filename}.tmp" | awk '{print $1}')
-  if [[ "$expected" = "$actual" ]]; then
-    return 0
-  else
-    return 1
-  fi
-}
-
-for i in 8 9 10 11 12 13 14;
-do
-  downloadRelease $i && clean_old_releases $i
-done
-