homebrew-bottles: migrate to ghcr.io

Ref to https://github.com/ustclug/ustcmirror-images/blob/master/homebrew-bottles/bottles-json/src/main.rs

use tmpdir for atomic update

clean old script and docker image

.github/workflows/docker-images.yml

@@ -18,7 +18,6 @@ jobs:
         image: 
           - bandersnatch
           - ftpsync
-          - homebrew-mirror
           - nix-channels
           - pub-mirror
           - rustup-mirror

dockerfiles/homebrew-mirror/Dockerfile

@@ -1,9 +0,0 @@
-FROM alpine:3.11
-
-RUN apk add --no-cache bash git curl ncurses ruby ruby-bigdecimal ruby-fiddle ruby-json ruby-etc ruby-irb ruby-json ruby-test-unit
-
-RUN git clone --depth 1 https://github.com/Homebrew/brew.git /home/homebrew/.linuxbrew/homebrew && cd /home/homebrew/.linuxbrew && mkdir bin && ln -s /home/homebrew/.linuxbrew/homebrew/bin/brew /home/homebrew/.linuxbrew/bin
-
-RUN git clone --depth 1 https://github.com/gaoyifan/homebrew-bottle-mirror.git /home/homebrew/.linuxbrew/homebrew/Library/Taps/gaoyifan/homebrew-bottle-mirror
-
-RUN chown -R 2000 /home/homebrew

homebrew-bottles.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+import os
+import json
+import requests
+import time
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+
+# mainly from apt-sync.py
+
+HOMEBREW_BOTTLE_DOMAIN = os.getenv("TUNASYNC_UPSTREAM_URL", "https://formulae.brew.sh/api/formula.json")
+WORKING_DIR = Path(os.getenv("TUNASYNC_WORKING_DIR", "/data"))
+DOWNLOAD_TIMEOUT=int(os.getenv('DOWNLOAD_TIMEOUT', '1800'))
+
+headers = {
+    "Accept": "application/vnd.oci.image.index.v1+json",
+    "Authorization": "Bearer QQ=="
+}
+
+def bottles():
+    b = {}
+    r = requests.get(HOMEBREW_BOTTLE_DOMAIN, timeout=(5, 10))
+    r.raise_for_status()
+    # refer to https://github.com/ustclug/ustcmirror-images/blob/master/homebrew-bottles/bottles-json/src/main.rs
+    formulae = r.json()
+    for formula in formulae:
+        if formula["versions"]["bottle"] and "stable" in formula["bottle"]:
+            bs = formula["bottle"]["stable"]
+            for (platform, v) in bs["files"].items():
+                sha256 = v["sha256"]
+                url = v["url"]
+                name = formula["name"]
+                version = formula["versions"]["stable"]
+                revision = "" if formula["revision"] == 0 else f"_{formula['revision']}"
+                rebuild = "" if bs["rebuild"] == 0 else f".{bs['rebuild']}"
+                b[sha256] = {
+                    "url": url,
+                    "file": f"{name}-{version}{revision}.{platform}.bottle{rebuild}.tar.gz"
+                }
+    return b
+
+# borrowed from apt-sync.py
+def check_and_download(url: str, dst_file: Path, dst_tmp_file: Path):
+    if dst_file.is_file(): return 2 # old file
+    try:
+        start = time.time()
+        with requests.get(url, stream=True, timeout=(5, 10), headers=headers) as r:
+            r.raise_for_status()
+            if 'last-modified' in r.headers:
+                remote_ts = parsedate_to_datetime(
+                    r.headers['last-modified']).timestamp()
+            else: remote_ts = None
+
+            with dst_tmp_file.open('wb') as f:
+                for chunk in r.iter_content(chunk_size=1024**2):
+                    if time.time() - start > DOWNLOAD_TIMEOUT:
+                        raise TimeoutError("Download timeout")
+                    if not chunk: continue # filter out keep-alive new chunks
+
+                    f.write(chunk)
+            if remote_ts is not None:
+                os.utime(dst_tmp_file, (remote_ts, remote_ts))
+        return 0
+    except BaseException as e:
+        print(e, flush=True)
+        if dst_tmp_file.is_file(): dst_tmp_file.unlink()
+    return 1
+
+if __name__ == "__main__":
+    # clean tmp file from previous sync
+    TMP_DIR = WORKING_DIR / ".tmp"
+    TMP_DIR.mkdir(exist_ok=True)
+    for file in TMP_DIR.glob("*.tar.gz"):
+        print(f"Clean tmp file {file.name}", flush=True)
+        file.unlink()
+
+    b = bottles()
+    for sha256 in b:
+        file = b[sha256]["file"]
+
+        # dark magic for linuxbrew-bottles
+        if "https://formulae.brew.sh/api/formula-linux.json" == HOMEBREW_BOTTLE_DOMAIN and\
+            "x86_64_linux" not in file:
+            print(f"Ignoring {file}", flush=True)
+            continue
+
+        print(f"Downloading {file}", flush=True)
+        dst_file = WORKING_DIR / file
+        dst_tmp_file = TMP_DIR / file
+        ret = check_and_download(b[sha256]["url"], dst_file, dst_tmp_file)
+        if ret == 0:
+            dst_tmp_file.rename(dst_file)
+            print(f"Downloaded {file}", flush=True)
+        elif ret == 2:
+            print(f"Exists {file}, Skip", flush=True)
+
+    files = list(map(lambda x: x["file"], b.values()))
+    # garbage collection
+    for file in WORKING_DIR.glob("*.tar.gz"):
+        if file.name not in files:
+            print(f"GC {file.name}", flush=True)
+            file.unlink()

linuxbrew-bottles.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-# Run in Docker image: tunathu/homebrew-mirror
-set -e
-set -o pipefail
-
-export HOMEBREW_REPO=${HOMEBREW_REPO:-"https://github.com/Homebrew"}
-export HOMEBREW_TAPS=${HOMEBREW_TAPS:-"core"}
-export HOMEBREW_BOTTLE_DOMAIN=${TUNASYNC_UPSTREAM_URL:-"https://linuxbrew.bintray.com"}
-
-# Refer to https://github.com/gaoyifan/homebrew-bottle-mirror/blob/master/.docker/run
-
-for tap in $HOMEBREW_TAPS;
-do
-    if [[ "$tap" = core ]];then # special case for homebrew-core
-        export HOMEBREW_CACHE="${TUNASYNC_WORKING_DIR}/bottles"
-    else
-        export HOMEBREW_CACHE="${TUNASYNC_WORKING_DIR}/bottles-$tap"
-        export HOMEBREW_TAP="$tap"
-    fi
-    if [[ "$RUN_LINUXBREW" = true ]];then
-        repo_name="linuxbrew-${tap}"
-        args="linux"
-    else
-        repo_name="homebrew-${tap}"
-        args="mac"
-    fi
-    mkdir -p "$HOMEBREW_CACHE"
-    remote_filelist="$HOMEBREW_CACHE/filelist.txt"
-
-    echo "===== SYNC STARTED AT $(date -R) ====="
-    dir_core=/home/homebrew/.linuxbrew/homebrew/Library/Taps/homebrew/homebrew-core
-    rm -fr "$dir_core" &>/dev/null || true
-    echo "> update package info from $HOMEBREW_REPO/$repo_name.git..."
-    git clone --depth 1 "$HOMEBREW_REPO/$repo_name.git" "$dir_core"
-    echo ""
-    echo "> RUN brew bottle-mirror $args..."
-    /home/homebrew/.linuxbrew/bin/brew bottle-mirror "$args"
-    if [[ -f "$remote_filelist" ]];then # clean outdated files
-        local_filelist=/tmp/filelist.local
-        (cd ${HOMEBREW_CACHE}; find . -type f -iname "*.tmp" -delete)
-        (cd ${HOMEBREW_CACHE}; find . -type f -mtime 30 -iname "*.tar.gz") | sed 's+^\./++' > $local_filelist
-        comm <(sort $remote_filelist) <(sort $local_filelist) -13 | while read file; do
-            echo "deleting ${HOMEBREW_CACHE}/${file}"
-            rm "${HOMEBREW_CACHE}/${file}"
-        done
-    fi
-done