mirror of
https://github.com/tuna/tunasync-scripts.git
synced 2025-04-20 04:12:42 +00:00
use python to sync docker-ce
This commit is contained in:
bigeagle
parent
11a8825443
commit
7bd1297d6a
142
docker-ce.py
Normal file
142
docker-ce.py
Normal file
@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import subprocess as sp
|
||||
from pathlib import Path
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
import requests
|
||||
from pyquery import PyQuery as pq
|
||||
|
||||
|
||||
BASE_URL = os.getenv("TUNASYNC_UPSTREAM_URL", "https://download.docker.com/")
|
||||
WORKING_DIR = os.getenv("TUNASYNC_WORKING_DIR")
|
||||
|
||||
|
||||
class RemoteSite:
|
||||
|
||||
def __init__(self, base_url=BASE_URL):
|
||||
if not base_url.endswith('/'):
|
||||
base_url = base_url + '/'
|
||||
self.base_url = base_url
|
||||
self.meta_urls = []
|
||||
|
||||
def is_metafile_url(self, url):
|
||||
deb_dists=('debian', 'ubuntu', 'raspbian')
|
||||
rpm_dists=('fedora', 'centos')
|
||||
|
||||
for dist in deb_dists:
|
||||
if '/'+dist+'/' not in url:
|
||||
continue
|
||||
if '/Contents-' in url:
|
||||
return True
|
||||
if '/binary-' in url:
|
||||
return True
|
||||
if 'Release' in url:
|
||||
return True
|
||||
|
||||
for dist in rpm_dists:
|
||||
if '/'+dist+'/' not in url:
|
||||
continue
|
||||
if '/repodata/' in url:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def recursive_get_filelist(self, base_url, filter_meta=False):
|
||||
if not base_url.endswith('/'):
|
||||
yield base_url
|
||||
return
|
||||
|
||||
r = requests.get(base_url)
|
||||
if not r.ok:
|
||||
return
|
||||
|
||||
d = pq(r.text)
|
||||
for link in d('a'):
|
||||
if link.text.startswith('..'):
|
||||
continue
|
||||
href = base_url + link.text
|
||||
if filter_meta and self.is_metafile_url(href):
|
||||
self.meta_urls.append(href)
|
||||
elif link.text.endswith('/'):
|
||||
yield from self.recursive_get_filelist(href, filter_meta=filter_meta)
|
||||
else:
|
||||
yield href
|
||||
|
||||
def relpath(self, url):
|
||||
assert url.startswith(self.base_url)
|
||||
return url[len(self.base_url):]
|
||||
|
||||
@property
|
||||
def files(self):
|
||||
yield from self.recursive_get_filelist(self.base_url, filter_meta=True)
|
||||
for url in self.meta_urls:
|
||||
yield from self.recursive_get_filelist(url, filter_meta=False)
|
||||
|
||||
|
||||
def curl_download(remote_url: str, dst_file: Path):
|
||||
sp.check_call([
|
||||
"curl", "-o", str(dst_file),
|
||||
"-sL", "--remote-time", "--show-error",
|
||||
"--fail", remote_url,
|
||||
])
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", default=BASE_URL)
|
||||
parser.add_argument("--working-dir", default=WORKING_DIR)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.working_dir is None:
|
||||
raise Exception("Working Directory is None")
|
||||
|
||||
working_dir = Path(args.working_dir)
|
||||
|
||||
remote_filelist = []
|
||||
rs = RemoteSite(args.base_url)
|
||||
for url in rs.files:
|
||||
dst_file = working_dir / rs.relpath(url)
|
||||
remote_filelist.append(dst_file.relative_to(working_dir))
|
||||
|
||||
if dst_file.is_file():
|
||||
r = requests.head(url)
|
||||
remote_filesize = int(r.headers['content-length'])
|
||||
remote_date = parsedate_to_datetime(r.headers['last-modified'])
|
||||
stat = dst_file.stat()
|
||||
local_filesize = stat.st_size
|
||||
local_mtime = stat.st_mtime
|
||||
|
||||
if remote_filesize == local_filesize and remote_date.timestamp() == local_mtime:
|
||||
print("Skipping", dst_file.relative_to(working_dir), flush=True)
|
||||
continue
|
||||
|
||||
dst_file.unlink()
|
||||
else:
|
||||
dst_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print("downloading", url, flush=True)
|
||||
try:
|
||||
curl_download(url, dst_file)
|
||||
except Exception:
|
||||
print("Failed to download", url, flush=True)
|
||||
if dst_file.is_file():
|
||||
dst_file.unlink()
|
||||
|
||||
local_filelist = []
|
||||
for local_file in working_dir.glob('**/*'):
|
||||
if local_file.is_file():
|
||||
local_filelist.append(local_file.relative_to(working_dir))
|
||||
|
||||
for old_file in set(local_filelist) - set(remote_filelist):
|
||||
print("deleting", old_file, flush=True)
|
||||
old_file = working_dir / old_file
|
||||
old_file.unlink()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
# vim: ts=4 sw=4 sts=4 expandtab
|
||||
65
docker-ce.sh
65
docker-ce.sh
@ -1,65 +0,0 @@
|
||||
#!/bin/bash
|
||||
# requires: lftp, jq, python3.5, lxml, pyquery
|
||||
# set -x
|
||||
set -e
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
_here=`dirname $(realpath $0)`
|
||||
GET_FILELIST="${_here}/helpers/docker-ce-filelist.py"
|
||||
|
||||
BASE_URL="${TUNASYNC_UPSTREAM_URL:-"https://download.docker.com/"}"
|
||||
|
||||
REMOTE_FILELIST="${TUNASYNC_WORKING_DIR}/.filelist.remote"
|
||||
LOCAL_FILELIST="${TUNASYNC_WORKING_DIR}/.filelist.local"
|
||||
[[ -f $REMOTE_FILELIST ]] && rm $REMOTE_FILELIST
|
||||
[[ -f $LOCAL_FILELIST ]] && rm $LOCAL_FILELIST
|
||||
|
||||
function cleanup () {
|
||||
echo "cleaning up"
|
||||
[[ -f $REMOTE_FILELIST ]] && rm $REMOTE_FILELIST || true
|
||||
[[ -f $LOCAL_FILELIST ]] && rm $LOCAL_FILELIST || true
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
# download
|
||||
while read remote_url; do
|
||||
dst_rel_file=${remote_url#$BASE_URL}
|
||||
dst_file="${TUNASYNC_WORKING_DIR}/${dst_rel_file}"
|
||||
dst_dir=`dirname ${dst_file}`
|
||||
|
||||
echo "${dst_rel_file}" >> $REMOTE_FILELIST
|
||||
|
||||
if [[ -f ${dst_file} ]]; then
|
||||
remote_meta=`curl -sI "${remote_url}"`
|
||||
remote_filesize=`echo -e "$remote_meta" | grep -i '^content-length:' | awk '{print $2}' | tr -d '\n\r' || echo 0`
|
||||
remote_date=`echo -e "$remote_meta" | grep -i '^last-modified:' | sed 's/^last-modified: //I' | tr -d '\n\r' || echo 0`
|
||||
remote_date=`date --date="${remote_date}" +%s`
|
||||
|
||||
local_filesize=`stat -c "%s" ${dst_file}`
|
||||
local_date=`stat -c "%Y" ${dst_file}`
|
||||
|
||||
if (( ${remote_filesize} == ${local_filesize} && ${remote_date} == ${local_date} )) ; then
|
||||
echo "skipping ${dst_rel_file}"
|
||||
continue
|
||||
fi
|
||||
rm $dst_file
|
||||
else
|
||||
mkdir -p $dst_dir
|
||||
fi
|
||||
|
||||
echo "downloading ${remote_url}"
|
||||
curl -o ${dst_file} -s -L --remote-time --show-error --fail ${remote_url} || {
|
||||
echo "Failed: ${remote_url}"
|
||||
[[ -f ${dst_file} ]] && rm ${dst_file}
|
||||
}
|
||||
done < <($GET_FILELIST $BASE_URL)
|
||||
|
||||
# remove old files
|
||||
(cd ${TUNASYNC_WORKING_DIR}; find . -type f ) | sed 's+^\./++' > ${LOCAL_FILELIST}
|
||||
comm <(sort $REMOTE_FILELIST) <(sort $LOCAL_FILELIST) -13 | while read file; do
|
||||
file="${TUNASYNC_WORKING_DIR}/$file"
|
||||
echo "deleting ${file}"
|
||||
[[ -f $file ]] && rm ${file}
|
||||
done
|
||||
Loading…
x
Reference in New Issue
Block a user