yacloud/tunasync-scripts
1
0
Fork 0
mirror of https://github.com/tuna/tunasync-scripts.git synced 2025-04-20 04:12:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

更正link处理逻辑

Browse Source
This commit is contained in:
jimorsm 2024-01-19 15:31:13 +08:00
parent db2813cb6f
commit 7b1d7701c6
1 changed files with 58 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
github-release.py
View File

@ -21,21 +21,20 @@ REPOS = []
TIMEOUT_OPTION = (7, 10) TIMEOUT_OPTION = (7, 10)
total_size = 0 total_size = 0
def sizeof_fmt(num, suffix='iB'):
def sizeof_fmt(num, suffix="iB"): for unit in ['','K','M','G','T','P','E','Z']:
for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
if abs(num) < 1024.0: if abs(num) < 1024.0:
return "%3.2f%s%s" % (num, unit, suffix) return "%3.2f%s%s" % (num, unit, suffix)
num /= 1024.0 num /= 1024.0
return "%.2f%s%s" % (num, "Y", suffix) return "%.2f%s%s" % (num, 'Y', suffix)
# wrap around requests.get to use token if available # wrap around requests.get to use token if available
def github_get(*args, **kwargs): def github_get(*args, **kwargs):
headers = kwargs["headers"] if "headers" in kwargs else {} headers = kwargs['headers'] if 'headers' in kwargs else {}
if "GITHUB_TOKEN" in os.environ: if 'GITHUB_TOKEN' in os.environ:
headers["Authorization"] = "token {}".format(os.environ["GITHUB_TOKEN"]) headers['Authorization'] = 'token {}'.format(
kwargs["headers"] = headers os.environ['GITHUB_TOKEN'])
kwargs['headers'] = headers
return requests.get(*args, **kwargs) return requests.get(*args, **kwargs)
@ -45,12 +44,7 @@ def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size:
r.raise_for_status() r.raise_for_status()
tmp_dst_file = None tmp_dst_file = None
try: try:
with tempfile.NamedTemporaryFile( with tempfile.NamedTemporaryFile(prefix="." + dst_file.name + ".", suffix=".tmp", dir=dst_file.parent, delete=False) as f:
prefix="." + dst_file.name + ".",
suffix=".tmp",
dir=dst_file.parent,
delete=False,
) as f:
tmp_dst_file = Path(f.name) tmp_dst_file = Path(f.name)
for chunk in r.iter_content(chunk_size=1024**2): for chunk in r.iter_content(chunk_size=1024**2):
if chunk: # filter out keep-alive new chunks if chunk: # filter out keep-alive new chunks
@ -59,9 +53,7 @@ def do_download(remote_url: str, dst_file: Path, remote_ts: float, remote_size:
# check for downloaded size # check for downloaded size
downloaded_size = tmp_dst_file.stat().st_size downloaded_size = tmp_dst_file.stat().st_size
if remote_size != -1 and downloaded_size != remote_size: if remote_size != -1 and downloaded_size != remote_size:
raise Exception( raise Exception(f'File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes')
f"File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes"
)
os.utime(tmp_dst_file, (remote_ts, remote_ts)) os.utime(tmp_dst_file, (remote_ts, remote_ts))
tmp_dst_file.chmod(0o644) tmp_dst_file.chmod(0o644)
tmp_dst_file.replace(dst_file) tmp_dst_file.replace(dst_file)
@ -79,7 +71,8 @@ def downloading_worker(q):
url, dst_file, working_dir, updated, remote_size = item url, dst_file, working_dir, updated, remote_size = item
print("downloading", url, "to", dst_file.relative_to(working_dir), flush=True) print("downloading", url, "to",
dst_file.relative_to(working_dir), flush=True)
try: try:
do_download(url, dst_file, updated, remote_size) do_download(url, dst_file, updated, remote_size)
except Exception: except Exception:
@ -99,29 +92,24 @@ def create_workers(n):
def ensure_safe_name(filename): def ensure_safe_name(filename):
filename = filename.replace("\0", " ") filename = filename.replace('\0', ' ')
if filename == ".": if filename == '.':
return " ." return ' .'
elif filename == "..": elif filename == '..':
return ". ." return '. .'
else: else:
return filename.replace("/", "\\").replace("\\", "_") return filename.replace('/', '\\').replace('\\', '_')
def main(): def main():
import argparse import argparse
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default=BASE_URL) parser.add_argument("--base-url", default=BASE_URL)
parser.add_argument("--working-dir", default=WORKING_DIR) parser.add_argument("--working-dir", default=WORKING_DIR)
parser.add_argument( parser.add_argument("--workers", default=1, type=int,
"--workers", default=1, type=int, help="number of concurrent downloading jobs" help='number of concurrent downloading jobs')
) parser.add_argument("--fast-skip", action='store_true',
parser.add_argument( help='do not verify size and timestamp of existing files')
"--fast-skip",
action="store_true",
help="do not verify size and timestamp of existing files",
)
parser.add_argument("--config", default=CONFIG) parser.add_argument("--config", default=CONFIG)
args = parser.parse_args() args = parser.parse_args()
@ -140,11 +128,10 @@ def main():
global total_size global total_size
if tarball: if tarball:
url = release["tarball_url"] url = release['tarball_url']
updated = datetime.strptime( updated = datetime.strptime(
release["published_at"], "%Y-%m-%dT%H:%M:%SZ" release['published_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp()
).timestamp() dst_file = release_dir / 'repo-snapshot.tar.gz'
dst_file = release_dir / "repo-snapshot.tar.gz"
remote_filelist.append(dst_file.relative_to(working_dir)) remote_filelist.append(dst_file.relative_to(working_dir))
if dst_file.is_file(): if dst_file.is_file():
@ -154,21 +141,19 @@ def main():
# tarball has no size information, use -1 to skip size check # tarball has no size information, use -1 to skip size check
task_queue.put((url, dst_file, working_dir, updated, -1)) task_queue.put((url, dst_file, working_dir, updated, -1))
for asset in release["assets"]: for asset in release['assets']:
url = asset["browser_download_url"] url = asset['browser_download_url']
updated = datetime.strptime( updated = datetime.strptime(
asset["updated_at"], "%Y-%m-%dT%H:%M:%SZ" asset['updated_at'], '%Y-%m-%dT%H:%M:%SZ').timestamp()
).timestamp() dst_file = release_dir / ensure_safe_name(asset['name'])
dst_file = release_dir / ensure_safe_name(asset["name"])
remote_filelist.append(dst_file.relative_to(working_dir)) remote_filelist.append(dst_file.relative_to(working_dir))
remote_size = asset["size"] remote_size = asset['size']
total_size += remote_size total_size += remote_size
if dst_file.is_file(): if dst_file.is_file():
if args.fast_skip: if args.fast_skip:
print( print("fast skipping", dst_file.relative_to(
"fast skipping", dst_file.relative_to(working_dir), flush=True working_dir), flush=True)
)
continue continue
else: else:
stat = dst_file.stat() stat = dst_file.stat()
@ -176,12 +161,10 @@ def main():
local_mtime = stat.st_mtime local_mtime = stat.st_mtime
# print(f"{local_filesize} vs {asset['size']}") # print(f"{local_filesize} vs {asset['size']}")
# print(f"{local_mtime} vs {updated}") # print(f"{local_mtime} vs {updated}")
if ( if local_mtime > updated or \
local_mtime > updated remote_size == local_filesize and local_mtime == updated:
or remote_size == local_filesize print("skipping", dst_file.relative_to(
and local_mtime == updated working_dir), flush=True)
):
print("skipping", dst_file.relative_to(working_dir), flush=True)
continue continue
else: else:
dst_file.parent.mkdir(parents=True, exist_ok=True) dst_file.parent.mkdir(parents=True, exist_ok=True)
@ -221,26 +204,25 @@ def main():
try: try:
headers = {"Accept": "application/vnd.github+json"} headers = {"Accept": "application/vnd.github+json"}
r = github_get( releases = []
f"{args.base_url}{repo}/releases?per_page=100", headers=headers url_str = f"{args.base_url}{repo}/releases"
) while url_str:
r.raise_for_status() r = github_get(url_str, headers=headers)
releases = r.json()
next_url = r.headers["link"]
if next_url:
for url_str in next_url.split(","):
r = github_get(url_str.split(";")[0].strip()[1:-1], headers=headers)
r.raise_for_status() r.raise_for_status()
releases.extend(r.json()) releases.extend(r.json())
next_url = [ s.split(";")[0].strip()[1:-1] for s in r.headers["link"].split(",") if s.split(";")[1].strip() == 'rel="next"' ]
if next_url:
url_str = next_url[0]
else:
url_str = None
except: except:
traceback.print_exc() traceback.print_exc()
break break
n_downloaded = 0 n_downloaded = 0
for release in releases: for release in releases:
if not release["draft"] and (prerelease or not release["prerelease"]): if not release['draft'] and (prerelease or not release['prerelease']):
name = ensure_safe_name(release["name"] or release["tag_name"]) name = ensure_safe_name(release['name'] or release['tag_name'])
if len(name) == 0: if len(name) == 0:
print("Error: Unnamed release") print("Error: Unnamed release")
continue continue
@ -265,7 +247,7 @@ def main():
if cleaning: if cleaning:
local_filelist = [] local_filelist = []
for local_file in working_dir.glob("**/*"): for local_file in working_dir.glob('**/*'):
if local_file.is_file(): if local_file.is_file():
local_filelist.append(local_file.relative_to(working_dir)) local_filelist.append(local_file.relative_to(working_dir))
@ -274,7 +256,7 @@ def main():
old_file = working_dir / old_file old_file = working_dir / old_file
old_file.unlink() old_file.unlink()
for local_dir in working_dir.glob("*/*/*"): for local_dir in working_dir.glob('*/*/*'):
if local_dir.is_dir(): if local_dir.is_dir():
try: try:
# remove empty dirs only # remove empty dirs only
@ -284,7 +266,6 @@ def main():
print("Total size is", sizeof_fmt(total_size, suffix="")) print("Total size is", sizeof_fmt(total_size, suffix=""))
if __name__ == "__main__": if __name__ == "__main__":
main() main()