yacloud/tunasync-scripts
1
0
Fork 0
mirror of https://github.com/tuna/tunasync-scripts.git synced 2025-04-20 04:12:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

nix-channels: update

Browse Source 
https://discourse.nixos.org/t/announcement-moving-nixos-org-to-netlify/6212
This commit is contained in:
dramforever 2020-03-23 20:33:02 +08:00
parent 0babe7557c
commit a14304c1b2
2 changed files with 21 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dockerfiles/nix-channels/Dockerfile
View File

@ -1,7 +1,7 @@
FROM python:3-buster FROM python:3-buster
MAINTAINER Wang Ruikang <dramforever@live.com> MAINTAINER Wang Ruikang <dramforever@live.com>
RUN pip3 install pyquery requests && \ RUN pip3 install pyquery requests minio && \
# Install Nix. To simplify management we only copy binaries and create # Install Nix. To simplify management we only copy binaries and create
# symlinks, and do no further configuration # symlinks, and do no further configuration
curl https://mirrors.tuna.tsinghua.edu.cn/nix/nix-2.3.2/nix-2.3.2-x86_64-linux.tar.xz -o /tmp/nix.tar.xz && \ curl https://mirrors.tuna.tsinghua.edu.cn/nix/nix-2.3.2/nix-2.3.2-x86_64-linux.tar.xz -o /tmp/nix.tar.xz && \

52
nix-channels.py
View File

@ -3,11 +3,13 @@ import hashlib
import json import json
import logging import logging
import lzma import lzma
import minio
import os import os
import pytz
import re import re
import sys
import requests import requests
import subprocess import subprocess
import sys
from pyquery import PyQuery as pq from pyquery import PyQuery as pq
from datetime import datetime, timedelta from datetime import datetime, timedelta
@ -35,7 +37,12 @@ RETAIN_DAYS = float(os.getenv('NIX_MIRROR_RETAIN_DAYS', 30))
STORE_DIR = 'store' STORE_DIR = 'store'
RELEASES_DIR = 'releases' RELEASES_DIR = 'releases'
CLONE_SINCE = datetime(2018, 12, 1)
# Channels that have not updated since migration to Netlify [1] are assumed to
# be too old and defunct.
#
# [1]: https://discourse.nixos.org/t/announcement-moving-nixos-org-to-netlify/6212
CLONE_SINCE = datetime(2020, 3, 6, tzinfo=pytz.utc)
TIMEOUT = 60 TIMEOUT = 60
working_dir = Path(WORKING_DIR) working_dir = Path(WORKING_DIR)
@ -63,9 +70,6 @@ logging.basicConfig(
# Don't forget 'global failure' # Don't forget 'global failure'
failure = False failure = False
def http_head(*args, **kwargs):
return session.head(*args, timeout=TIMEOUT, **kwargs)
def http_get(*args, **kwargs): def http_get(*args, **kwargs):
return session.get(*args, timeout=TIMEOUT, **kwargs) return session.get(*args, timeout=TIMEOUT, **kwargs)
@ -131,28 +135,14 @@ def download(url, dest):
download_dest.rename(dest) download_dest.rename(dest)
def get_links(url): client = minio.Minio('s3.amazonaws.com')
r = http_get(url)
r.raise_for_status()
node = pq(r.content) def get_channels():
return [
links = [] (x.object_name, x.last_modified)
for row in node('tr'): for x in client.list_objects_v2('nix-channels')
td = pq(row)('td') if re.fullmatch(r'(nixos|nixpkgs)-.+[^/]', x.object_name)
if len(td) != 5: ]
continue
link_target = td[1].find('a').get('href')
if link_target.startswith('/'):
# Link to parent directory
continue
last_updated = td[2].text.strip()
links.append((link_target, last_updated))
return links
def clone_channels(): def clone_channels():
logging.info(f'- Fetching channels') logging.info(f'- Fetching channels')
@ -161,17 +151,15 @@ def clone_channels():
working_dir.mkdir(parents=True, exist_ok=True) working_dir.mkdir(parents=True, exist_ok=True)
for channel, chan_updated in get_links(f'{UPSTREAM_URL}/'): for channel, chan_updated in get_channels():
chan_path = working_dir / channel chan_path = working_dir / channel
# Old channels, little value in cloning and format changes # Old channels, little value in cloning and format changes
if datetime.strptime(chan_updated, '%Y-%m-%d %H:%M') < CLONE_SINCE: if chan_updated < CLONE_SINCE:
continue continue
chan_redirect_res = http_head(f'{UPSTREAM_URL}/{channel}', allow_redirects=False) chan_obj = client.get_object('nix-channels', channel)
chan_redirect_res.raise_for_status() chan_location = chan_obj.headers['x-amz-website-redirect-location']
chan_location = chan_redirect_res.headers['Location']
chan_release = chan_location.split('/')[-1] chan_release = chan_location.split('/')[-1]