mirror of
https://github.com/tuna/tunasync-scripts.git
synced 2025-07-01 07:22:45 +00:00
homebrew-bottles: migrate to ghcr.io
Ref to https://github.com/ustclug/ustcmirror-images/blob/master/homebrew-bottles/bottles-json/src/main.rs use tmpdir for atomic update clean old script and docker image
This commit is contained in:
Zenithal (at bravo)
Zenithal (at neo)
parent
2543b0fc0b
commit
7c5b56b6eb
1
.github/workflows/docker-images.yml
vendored
1
.github/workflows/docker-images.yml
vendored
@ -18,7 +18,6 @@ jobs:
|
||||
image:
|
||||
- bandersnatch
|
||||
- ftpsync
|
||||
- homebrew-mirror
|
||||
- nix-channels
|
||||
- pub-mirror
|
||||
- rustup-mirror
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
FROM alpine:3.11
|
||||
|
||||
RUN apk add --no-cache bash git curl ncurses ruby ruby-bigdecimal ruby-fiddle ruby-json ruby-etc ruby-irb ruby-json ruby-test-unit
|
||||
|
||||
RUN git clone --depth 1 https://github.com/Homebrew/brew.git /home/homebrew/.linuxbrew/homebrew && cd /home/homebrew/.linuxbrew && mkdir bin && ln -s /home/homebrew/.linuxbrew/homebrew/bin/brew /home/homebrew/.linuxbrew/bin
|
||||
|
||||
RUN git clone --depth 1 https://github.com/gaoyifan/homebrew-bottle-mirror.git /home/homebrew/.linuxbrew/homebrew/Library/Taps/gaoyifan/homebrew-bottle-mirror
|
||||
|
||||
RUN chown -R 2000 /home/homebrew
|
||||
102
homebrew-bottles.py
Executable file
102
homebrew-bottles.py
Executable file
@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import json
|
||||
import requests
|
||||
import time
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
|
||||
# mainly from apt-sync.py
|
||||
|
||||
HOMEBREW_BOTTLE_DOMAIN = os.getenv("TUNASYNC_UPSTREAM_URL", "https://formulae.brew.sh/api/formula.json")
|
||||
WORKING_DIR = Path(os.getenv("TUNASYNC_WORKING_DIR", "/data"))
|
||||
DOWNLOAD_TIMEOUT=int(os.getenv('DOWNLOAD_TIMEOUT', '1800'))
|
||||
|
||||
headers = {
|
||||
"Accept": "application/vnd.oci.image.index.v1+json",
|
||||
"Authorization": "Bearer QQ=="
|
||||
}
|
||||
|
||||
def bottles():
|
||||
b = {}
|
||||
r = requests.get(HOMEBREW_BOTTLE_DOMAIN, timeout=(5, 10))
|
||||
r.raise_for_status()
|
||||
# refer to https://github.com/ustclug/ustcmirror-images/blob/master/homebrew-bottles/bottles-json/src/main.rs
|
||||
formulae = r.json()
|
||||
for formula in formulae:
|
||||
if formula["versions"]["bottle"] and "stable" in formula["bottle"]:
|
||||
bs = formula["bottle"]["stable"]
|
||||
for (platform, v) in bs["files"].items():
|
||||
sha256 = v["sha256"]
|
||||
url = v["url"]
|
||||
name = formula["name"]
|
||||
version = formula["versions"]["stable"]
|
||||
revision = "" if formula["revision"] == 0 else f"_{formula['revision']}"
|
||||
rebuild = "" if bs["rebuild"] == 0 else f".{bs['rebuild']}"
|
||||
b[sha256] = {
|
||||
"url": url,
|
||||
"file": f"{name}-{version}{revision}.{platform}.bottle{rebuild}.tar.gz"
|
||||
}
|
||||
return b
|
||||
|
||||
# borrowed from apt-sync.py
|
||||
def check_and_download(url: str, dst_file: Path, dst_tmp_file: Path):
|
||||
if dst_file.is_file(): return 2 # old file
|
||||
try:
|
||||
start = time.time()
|
||||
with requests.get(url, stream=True, timeout=(5, 10), headers=headers) as r:
|
||||
r.raise_for_status()
|
||||
if 'last-modified' in r.headers:
|
||||
remote_ts = parsedate_to_datetime(
|
||||
r.headers['last-modified']).timestamp()
|
||||
else: remote_ts = None
|
||||
|
||||
with dst_tmp_file.open('wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=1024**2):
|
||||
if time.time() - start > DOWNLOAD_TIMEOUT:
|
||||
raise TimeoutError("Download timeout")
|
||||
if not chunk: continue # filter out keep-alive new chunks
|
||||
|
||||
f.write(chunk)
|
||||
if remote_ts is not None:
|
||||
os.utime(dst_tmp_file, (remote_ts, remote_ts))
|
||||
return 0
|
||||
except BaseException as e:
|
||||
print(e, flush=True)
|
||||
if dst_tmp_file.is_file(): dst_tmp_file.unlink()
|
||||
return 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
# clean tmp file from previous sync
|
||||
TMP_DIR = WORKING_DIR / ".tmp"
|
||||
TMP_DIR.mkdir(exist_ok=True)
|
||||
for file in TMP_DIR.glob("*.tar.gz"):
|
||||
print(f"Clean tmp file {file.name}", flush=True)
|
||||
file.unlink()
|
||||
|
||||
b = bottles()
|
||||
for sha256 in b:
|
||||
file = b[sha256]["file"]
|
||||
|
||||
# dark magic for linuxbrew-bottles
|
||||
if "https://formulae.brew.sh/api/formula-linux.json" == HOMEBREW_BOTTLE_DOMAIN and\
|
||||
"x86_64_linux" not in file:
|
||||
print(f"Ignoring {file}", flush=True)
|
||||
continue
|
||||
|
||||
print(f"Downloading {file}", flush=True)
|
||||
dst_file = WORKING_DIR / file
|
||||
dst_tmp_file = TMP_DIR / file
|
||||
ret = check_and_download(b[sha256]["url"], dst_file, dst_tmp_file)
|
||||
if ret == 0:
|
||||
dst_tmp_file.rename(dst_file)
|
||||
print(f"Downloaded {file}", flush=True)
|
||||
elif ret == 2:
|
||||
print(f"Exists {file}, Skip", flush=True)
|
||||
|
||||
files = list(map(lambda x: x["file"], b.values()))
|
||||
# garbage collection
|
||||
for file in WORKING_DIR.glob("*.tar.gz"):
|
||||
if file.name not in files:
|
||||
print(f"GC {file.name}", flush=True)
|
||||
file.unlink()
|
||||
@ -1,47 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Run in Docker image: tunathu/homebrew-mirror
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
export HOMEBREW_REPO=${HOMEBREW_REPO:-"https://github.com/Homebrew"}
|
||||
export HOMEBREW_TAPS=${HOMEBREW_TAPS:-"core"}
|
||||
export HOMEBREW_BOTTLE_DOMAIN=${TUNASYNC_UPSTREAM_URL:-"https://linuxbrew.bintray.com"}
|
||||
|
||||
# Refer to https://github.com/gaoyifan/homebrew-bottle-mirror/blob/master/.docker/run
|
||||
|
||||
for tap in $HOMEBREW_TAPS;
|
||||
do
|
||||
if [[ "$tap" = core ]];then # special case for homebrew-core
|
||||
export HOMEBREW_CACHE="${TUNASYNC_WORKING_DIR}/bottles"
|
||||
else
|
||||
export HOMEBREW_CACHE="${TUNASYNC_WORKING_DIR}/bottles-$tap"
|
||||
export HOMEBREW_TAP="$tap"
|
||||
fi
|
||||
if [[ "$RUN_LINUXBREW" = true ]];then
|
||||
repo_name="linuxbrew-${tap}"
|
||||
args="linux"
|
||||
else
|
||||
repo_name="homebrew-${tap}"
|
||||
args="mac"
|
||||
fi
|
||||
mkdir -p "$HOMEBREW_CACHE"
|
||||
remote_filelist="$HOMEBREW_CACHE/filelist.txt"
|
||||
|
||||
echo "===== SYNC STARTED AT $(date -R) ====="
|
||||
dir_core=/home/homebrew/.linuxbrew/homebrew/Library/Taps/homebrew/homebrew-core
|
||||
rm -fr "$dir_core" &>/dev/null || true
|
||||
echo "> update package info from $HOMEBREW_REPO/$repo_name.git..."
|
||||
git clone --depth 1 "$HOMEBREW_REPO/$repo_name.git" "$dir_core"
|
||||
echo ""
|
||||
echo "> RUN brew bottle-mirror $args..."
|
||||
/home/homebrew/.linuxbrew/bin/brew bottle-mirror "$args"
|
||||
if [[ -f "$remote_filelist" ]];then # clean outdated files
|
||||
local_filelist=/tmp/filelist.local
|
||||
(cd ${HOMEBREW_CACHE}; find . -type f -iname "*.tmp" -delete)
|
||||
(cd ${HOMEBREW_CACHE}; find . -type f -mtime 30 -iname "*.tar.gz") | sed 's+^\./++' > $local_filelist
|
||||
comm <(sort $remote_filelist) <(sort $local_filelist) -13 | while read file; do
|
||||
echo "deleting ${HOMEBREW_CACHE}/${file}"
|
||||
rm "${HOMEBREW_CACHE}/${file}"
|
||||
done
|
||||
fi
|
||||
done
|
||||
Loading…
x
Reference in New Issue
Block a user