yacloud/tunasync-scripts
1
0
Fork 0
mirror of https://github.com/tuna/tunasync-scripts.git synced 2025-07-01 07:22:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

verify the content of ~5% of files each time, close #25

Browse Source
This commit is contained in:
z4yx 2020-03-07 11:43:23 +08:00
parent f268ce316e
commit e87a55da65
1 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
anaconda.py
View File

@ -3,6 +3,7 @@ import hashlib
import json import json
import logging import logging
import os import os
import random
import shutil import shutil
import subprocess as sp import subprocess as sp
import tempfile import tempfile
@ -169,10 +170,13 @@ def sync_installer(repo_url, local_dir: Path):
local_filesize = stat.st_size local_filesize = stat.st_size
local_mtime = stat.st_mtime local_mtime = stat.st_mtime
if remote_filesize == local_filesize and remote_date.timestamp() == local_mtime: # Do content verification on ~5% of files (see issue #25)
if remote_filesize == local_filesize and remote_date.timestamp() == local_mtime and\
(random.random() < 0.95 or md5_check(dst_file, md5)):
logging.info("Skipping {}".format(filename)) logging.info("Skipping {}".format(filename))
continue continue
logging.info("Removing {}".format(filename))
dst_file.unlink() dst_file.unlink()
for retry in range(3): for retry in range(3):
@ -197,14 +201,17 @@ def main():
raise Exception("Working Directory is None") raise Exception("Working Directory is None")
working_dir = Path(args.working_dir) working_dir = Path(args.working_dir)
random.seed()
for dist in ("archive", "miniconda"): if random.random() < 0.1: # Syncing installer less frequently
remote_url = "{}/{}".format(CONDA_REPO_BASE_URL, dist) logging.info("Syncing installers...")
local_dir = working_dir / dist for dist in ("archive", "miniconda"):
try: remote_url = "{}/{}".format(CONDA_REPO_BASE_URL, dist)
sync_installer(remote_url, local_dir) local_dir = working_dir / dist
except Exception: try:
logging.exception("Failed to sync installers of {}".format(dist)) sync_installer(remote_url, local_dir)
except Exception:
logging.exception("Failed to sync installers of {}".format(dist))
for repo in CONDA_REPOS: for repo in CONDA_REPOS:
for arch in CONDA_ARCHES: for arch in CONDA_ARCHES: