From 7c5b56b6eb48201d2ef4ada3fe06c64e2dc0f0b7 Mon Sep 17 00:00:00 2001 From: "Zenithal (at bravo)" Date: Fri, 10 Sep 2021 23:37:32 +0800 Subject: [PATCH] homebrew-bottles: migrate to ghcr.io Ref to https://github.com/ustclug/ustcmirror-images/blob/master/homebrew-bottles/bottles-json/src/main.rs use tmpdir for atomic update clean old script and docker image --- .github/workflows/docker-images.yml | 1 - dockerfiles/homebrew-mirror/Dockerfile | 9 --- homebrew-bottles.py | 102 +++++++++++++++++++++++++ linuxbrew-bottles.sh | 47 ------------ 4 files changed, 102 insertions(+), 57 deletions(-) delete mode 100644 dockerfiles/homebrew-mirror/Dockerfile create mode 100755 homebrew-bottles.py delete mode 100755 linuxbrew-bottles.sh diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index f5549e5..fbcc96b 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -18,7 +18,6 @@ jobs: image: - bandersnatch - ftpsync - - homebrew-mirror - nix-channels - pub-mirror - rustup-mirror diff --git a/dockerfiles/homebrew-mirror/Dockerfile b/dockerfiles/homebrew-mirror/Dockerfile deleted file mode 100644 index 2e84b41..0000000 --- a/dockerfiles/homebrew-mirror/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM alpine:3.11 - -RUN apk add --no-cache bash git curl ncurses ruby ruby-bigdecimal ruby-fiddle ruby-json ruby-etc ruby-irb ruby-json ruby-test-unit - -RUN git clone --depth 1 https://github.com/Homebrew/brew.git /home/homebrew/.linuxbrew/homebrew && cd /home/homebrew/.linuxbrew && mkdir bin && ln -s /home/homebrew/.linuxbrew/homebrew/bin/brew /home/homebrew/.linuxbrew/bin - -RUN git clone --depth 1 https://github.com/gaoyifan/homebrew-bottle-mirror.git /home/homebrew/.linuxbrew/homebrew/Library/Taps/gaoyifan/homebrew-bottle-mirror - -RUN chown -R 2000 /home/homebrew diff --git a/homebrew-bottles.py b/homebrew-bottles.py new file mode 100755 index 0000000..fe86e35 --- /dev/null +++ b/homebrew-bottles.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +import os +import json +import requests +import time +from email.utils import parsedate_to_datetime +from pathlib import Path + +# mainly from apt-sync.py + +HOMEBREW_BOTTLE_DOMAIN = os.getenv("TUNASYNC_UPSTREAM_URL", "https://formulae.brew.sh/api/formula.json") +WORKING_DIR = Path(os.getenv("TUNASYNC_WORKING_DIR", "/data")) +DOWNLOAD_TIMEOUT=int(os.getenv('DOWNLOAD_TIMEOUT', '1800')) + +headers = { + "Accept": "application/vnd.oci.image.index.v1+json", + "Authorization": "Bearer QQ==" +} + +def bottles(): + b = {} + r = requests.get(HOMEBREW_BOTTLE_DOMAIN, timeout=(5, 10)) + r.raise_for_status() + # refer to https://github.com/ustclug/ustcmirror-images/blob/master/homebrew-bottles/bottles-json/src/main.rs + formulae = r.json() + for formula in formulae: + if formula["versions"]["bottle"] and "stable" in formula["bottle"]: + bs = formula["bottle"]["stable"] + for (platform, v) in bs["files"].items(): + sha256 = v["sha256"] + url = v["url"] + name = formula["name"] + version = formula["versions"]["stable"] + revision = "" if formula["revision"] == 0 else f"_{formula['revision']}" + rebuild = "" if bs["rebuild"] == 0 else f".{bs['rebuild']}" + b[sha256] = { + "url": url, + "file": f"{name}-{version}{revision}.{platform}.bottle{rebuild}.tar.gz" + } + return b + +# borrowed from apt-sync.py +def check_and_download(url: str, dst_file: Path, dst_tmp_file: Path): + if dst_file.is_file(): return 2 # old file + try: + start = time.time() + with requests.get(url, stream=True, timeout=(5, 10), headers=headers) as r: + r.raise_for_status() + if 'last-modified' in r.headers: + remote_ts = parsedate_to_datetime( + r.headers['last-modified']).timestamp() + else: remote_ts = None + + with dst_tmp_file.open('wb') as f: + for chunk in r.iter_content(chunk_size=1024**2): + if time.time() - start > DOWNLOAD_TIMEOUT: + raise TimeoutError("Download timeout") + if not chunk: continue # filter out keep-alive new chunks + + f.write(chunk) + if remote_ts is not None: + os.utime(dst_tmp_file, (remote_ts, remote_ts)) + return 0 + except BaseException as e: + print(e, flush=True) + if dst_tmp_file.is_file(): dst_tmp_file.unlink() + return 1 + +if __name__ == "__main__": + # clean tmp file from previous sync + TMP_DIR = WORKING_DIR / ".tmp" + TMP_DIR.mkdir(exist_ok=True) + for file in TMP_DIR.glob("*.tar.gz"): + print(f"Clean tmp file {file.name}", flush=True) + file.unlink() + + b = bottles() + for sha256 in b: + file = b[sha256]["file"] + + # dark magic for linuxbrew-bottles + if "https://formulae.brew.sh/api/formula-linux.json" == HOMEBREW_BOTTLE_DOMAIN and\ + "x86_64_linux" not in file: + print(f"Ignoring {file}", flush=True) + continue + + print(f"Downloading {file}", flush=True) + dst_file = WORKING_DIR / file + dst_tmp_file = TMP_DIR / file + ret = check_and_download(b[sha256]["url"], dst_file, dst_tmp_file) + if ret == 0: + dst_tmp_file.rename(dst_file) + print(f"Downloaded {file}", flush=True) + elif ret == 2: + print(f"Exists {file}, Skip", flush=True) + + files = list(map(lambda x: x["file"], b.values())) + # garbage collection + for file in WORKING_DIR.glob("*.tar.gz"): + if file.name not in files: + print(f"GC {file.name}", flush=True) + file.unlink() diff --git a/linuxbrew-bottles.sh b/linuxbrew-bottles.sh deleted file mode 100755 index a8fb833..0000000 --- a/linuxbrew-bottles.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Run in Docker image: tunathu/homebrew-mirror -set -e -set -o pipefail - -export HOMEBREW_REPO=${HOMEBREW_REPO:-"https://github.com/Homebrew"} -export HOMEBREW_TAPS=${HOMEBREW_TAPS:-"core"} -export HOMEBREW_BOTTLE_DOMAIN=${TUNASYNC_UPSTREAM_URL:-"https://linuxbrew.bintray.com"} - -# Refer to https://github.com/gaoyifan/homebrew-bottle-mirror/blob/master/.docker/run - -for tap in $HOMEBREW_TAPS; -do - if [[ "$tap" = core ]];then # special case for homebrew-core - export HOMEBREW_CACHE="${TUNASYNC_WORKING_DIR}/bottles" - else - export HOMEBREW_CACHE="${TUNASYNC_WORKING_DIR}/bottles-$tap" - export HOMEBREW_TAP="$tap" - fi - if [[ "$RUN_LINUXBREW" = true ]];then - repo_name="linuxbrew-${tap}" - args="linux" - else - repo_name="homebrew-${tap}" - args="mac" - fi - mkdir -p "$HOMEBREW_CACHE" - remote_filelist="$HOMEBREW_CACHE/filelist.txt" - - echo "===== SYNC STARTED AT $(date -R) =====" - dir_core=/home/homebrew/.linuxbrew/homebrew/Library/Taps/homebrew/homebrew-core - rm -fr "$dir_core" &>/dev/null || true - echo "> update package info from $HOMEBREW_REPO/$repo_name.git..." - git clone --depth 1 "$HOMEBREW_REPO/$repo_name.git" "$dir_core" - echo "" - echo "> RUN brew bottle-mirror $args..." - /home/homebrew/.linuxbrew/bin/brew bottle-mirror "$args" - if [[ -f "$remote_filelist" ]];then # clean outdated files - local_filelist=/tmp/filelist.local - (cd ${HOMEBREW_CACHE}; find . -type f -iname "*.tmp" -delete) - (cd ${HOMEBREW_CACHE}; find . -type f -mtime 30 -iname "*.tar.gz") | sed 's+^\./++' > $local_filelist - comm <(sort $remote_filelist) <(sort $local_filelist) -13 | while read file; do - echo "deleting ${HOMEBREW_CACHE}/${file}" - rm "${HOMEBREW_CACHE}/${file}" - done - fi -done