From adeac6a96d8469e2359bf05c356114ef6024b607 Mon Sep 17 00:00:00 2001 From: sssxie Date: Wed, 20 Jan 2021 11:33:24 +0800 Subject: [PATCH] add HCC cloud repo & adjust code format --- anaconda.py | 136 ++++++++++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 63 deletions(-) diff --git a/anaconda.py b/anaconda.py index 69e869c..1241717 100755 --- a/anaconda.py +++ b/anaconda.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import argparse import hashlib import json import logging @@ -18,51 +19,52 @@ import requests DEFAULT_CONDA_REPO_BASE = "https://repo.continuum.io" DEFAULT_CONDA_CLOUD_BASE = "https://conda.anaconda.org" -CONDA_REPO_BASE_URL = os.getenv("CONDA_REPO_URL", "https://repo.continuum.io") -CONDA_CLOUD_BASE_URL = os.getenv("CONDA_COULD_URL", "https://conda.anaconda.org") +CONDA_REPO_BASE_URL = os.getenv("CONDA_REPO_URL", + "https://repo.continuum.io") +CONDA_CLOUD_BASE_URL = os.getenv("CONDA_COULD_URL", + "https://conda.anaconda.org") WORKING_DIR = os.getenv("TUNASYNC_WORKING_DIR") CONDA_REPOS = ("main", "free", "r", "msys2") CONDA_ARCHES = ( - "noarch", "linux-64", "linux-32", "linux-armv6l", "linux-armv7l", - "linux-ppc64le", "osx-64", "osx-32", "win-64", "win-32" + "linux-32", "linux-64", "linux-armv6l", "linux-armv7l", "linux-ppc64le", + "noarch", "osx-32", "osx-64", "win-32", "win-64" ) - CONDA_CLOUD_REPOS = ( - "conda-forge/linux-64", "conda-forge/osx-64", "conda-forge/win-64", "conda-forge/noarch", - "msys2/linux-64", "msys2/win-64", "msys2/noarch", - "rapidsai/linux-64", "rapidsai/noarch", - "bioconda/linux-64", "bioconda/osx-64", "bioconda/win-64", "bioconda/noarch", - "menpo/linux-64", "menpo/osx-64", "menpo/win-64", "menpo/win-32", "menpo/noarch", - "pytorch/linux-64", "pytorch/osx-64", "pytorch/win-64", "pytorch/win-32", "pytorch/noarch", - "pytorch-test/linux-64", "pytorch-test/osx-64", "pytorch-test/win-64", "pytorch-test/win-32", "pytorch-test/noarch", - "stackless/linux-64", "stackless/win-64", "stackless/win-32", "stackless/linux-32", "stackless/osx-64", "stackless/noarch", - "fermi/linux-64", "fermi/osx-64", "fermi/win-64", "fermi/noarch", - "fastai/linux-64", "fastai/osx-64", "fastai/win-64", "fastai/noarch", - "omnia/linux-64", "omnia/osx-64", "omnia/win-64", "omnia/noarch", - "simpleitk/linux-64", "simpleitk/linux-32", "simpleitk/osx-64", "simpleitk/win-64", "simpleitk/win-32", "simpleitk/noarch", - "caffe2/linux-64", "caffe2/osx-64", "caffe2/win-64", "caffe2/noarch", - "plotly/linux-64", "plotly/linux-32", "plotly/osx-64", "plotly/win-64", "plotly/win-32", "plotly/noarch", - "intel/linux-64", "intel/linux-32", "intel/osx-64", "intel/win-64", "intel/win-32", "intel/noarch", - "auto/linux-64", "auto/linux-32", "auto/osx-64", "auto/win-64", "auto/win-32", "auto/noarch", - "ursky/linux-64", "ursky/osx-64", "ursky/noarch", - "matsci/linux-64", "matsci/osx-64", "matsci/win-64", "matsci/noarch", - "psi4/linux-64", "psi4/osx-64", "psi4/win-64", "psi4/noarch", - "Paddle/linux-64", "Paddle/linux-32", "Paddle/osx-64", "Paddle/win-64", "Paddle/win-32", "Paddle/noarch", - "deepmodeling/linux-64", "deepmodeling/noarch", - "numba/linux-64", "numba/linux-32", "numba/osx-64", "numba/win-64", "numba/win-32", "numba/noarch", - "numba/label/dev/win-64", "numba/label/dev/noarch", - "pyviz/linux-64", "pyviz/linux-32", "pyviz/win-64", "pyviz/win-32", "pyviz/osx-64", "pyviz/noarch", - "dglteam/linux-64", "dglteam/win-64", "dglteam/osx-64", "dglteam/noarch", - "rdkit/linux-64", "rdkit/win-64", "rdkit/osx-64", "rdkit/noarch", - "mordred-descriptor/linux-64", "mordred-descriptor/win-64", "mordred-descriptor/win-32", "mordred-descriptor/osx-64", "mordred-descriptor/noarch", - "ohmeta/linux-64", "ohmeta/osx-64", "ohmeta/noarch", - "qiime2/linux-64", "qiime2/osx-64", "qiime2/noarch", - "biobakery/linux-64", "biobakery/osx-64", "biobakery/noarch", - "c4aarch64/linux-aarch64", "c4aarch64/noarch", - "pytorch3d/linux-64", "pytorch3d/noarch", - "idaholab/linux-64", "idaholab/noarch", + "Paddle", + "auto", + "biobakery", + "bioconda", + "c4aarch64", + "caffe2", + "conda-forge", + "deepmodeling", + "dglteam", + "fastai", + "fermi", + "hcc", + "idaholab", + "intel", + "matsci", + "menpo", + "mordred-descriptor", + "msys2", + "numba", + "ohmeta", + "omnia", + "plotly", + "psi4", + "pytorch", + "pytorch-test", + "pytorch3d", + "pyviz", + "qiime2", + "rapidsai", + "rdkit", + "simpleitk", + "stackless", + "ursky", ) EXCLUDED_PACKAGES = ( @@ -77,14 +79,16 @@ logging.basicConfig( format="[%(asctime)s] [%(levelname)s] %(message)s", ) -def sizeof_fmt(num, suffix='iB'): - for unit in ['','K','M','G','T','P','E','Z']: + +def sizeof_fmt(num, suffix: str = 'iB'): + for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: if abs(num) < 1024.0: return "%3.2f%s%s" % (num, unit, suffix) num /= 1024.0 return "%.2f%s%s" % (num, 'Y', suffix) -def md5_check(file: Path, md5: str = None): + +def md5_check(file: Path, md5: str = ''): m = hashlib.md5() with file.open('rb') as f: while True: @@ -95,7 +99,7 @@ def md5_check(file: Path, md5: str = None): return m.hexdigest() == md5 -def curl_download(remote_url: str, dst_file: Path, md5: str = None): +def curl_download(remote_url: str, dst_file: Path, md5: str = ''): sp.check_call([ "curl", "-o", str(dst_file), "-sL", "--remote-time", "--show-error", @@ -169,7 +173,6 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool): break logging.error("Failed to download {}: {}".format(filename, err)) - shutil.move(str(tmp_repodata), str(local_dir / "repodata.json")) shutil.move(str(tmp_bz2_repodata), str(local_dir / "repodata.json.bz2")) if tmp_current_repodata.is_file(): @@ -193,10 +196,11 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool): repodata_url, len(remote_filelist), sizeof_fmt(total_size))) return total_size -def sync_installer(repo_url, local_dir: Path): + +def sync_installer(repo_url: str, local_dir: Path): logging.info("Start syncing {}".format(repo_url)) local_dir.mkdir(parents=True, exist_ok=True) - full_scan = random.random() < 0.1 # Do full version check less frequently + full_scan = random.random() < 0.1 # Do full version check less frequently def remote_list(): r = requests.get(repo_url, timeout=TIMEOUT_OPTION) @@ -215,7 +219,8 @@ def sync_installer(repo_url, local_dir: Path): dst_file_wip = local_dir / ('.downloading.' + filename) if dst_file.is_file(): - r = requests.head(pkg_url, allow_redirects=True, timeout=TIMEOUT_OPTION) + r = requests.head(pkg_url, allow_redirects=True, + timeout=TIMEOUT_OPTION) len_avail = 'content-length' in r.headers if len_avail: remote_filesize = int(r.headers['content-length']) @@ -225,8 +230,9 @@ def sync_installer(repo_url, local_dir: Path): local_mtime = stat.st_mtime # Do content verification on ~5% of files (see issue #25) - if (not len_avail or remote_filesize == local_filesize) and remote_date.timestamp() == local_mtime and \ - (random.random() < 0.95 or md5_check(dst_file, md5)): + if ((not len_avail or remote_filesize == local_filesize) and + remote_date.timestamp() == local_mtime and + (random.random() < 0.95 or md5_check(dst_file, md5))): logging.info("Skipping {}".format(filename)) # Stop the scanning if the most recent version is present @@ -252,8 +258,8 @@ def sync_installer(repo_url, local_dir: Path): break logging.error("Failed to download {}: {}".format(filename, err)) + def main(): - import argparse parser = argparse.ArgumentParser() parser.add_argument("--working-dir", default=WORKING_DIR) parser.add_argument("--delete", action='store_true', @@ -280,33 +286,37 @@ def main(): for repo in CONDA_REPOS: for arch in CONDA_ARCHES: - remote_url = "{}/pkgs/{}/{}".format(CONDA_REPO_BASE_URL, repo, arch) + remote_url = "{}/pkgs/{}/{}".format(CONDA_REPO_BASE_URL, + repo, arch) local_dir = working_dir / "pkgs" / repo / arch tmpdir = tempfile.mkdtemp() try: - size_statistics += sync_repo(remote_url, - local_dir, Path(tmpdir), args.delete) + size_statistics += sync_repo(remote_url, local_dir, + Path(tmpdir), args.delete) except Exception: - logging.exception("Failed to sync repo: {}/{}".format(repo, arch)) + logging.exception( + "Failed to sync repo: {}/{}".format(repo, arch)) finally: shutil.rmtree(tmpdir) for repo in CONDA_CLOUD_REPOS: - remote_url = "{}/{}".format(CONDA_CLOUD_BASE_URL, repo) - local_dir = working_dir / "cloud" / repo + for arch in CONDA_ARCHES: + remote_url = "{}/{}/{}".format(CONDA_CLOUD_BASE_URL, repo, arch) + local_dir = working_dir / "cloud" / repo / arch - tmpdir = tempfile.mkdtemp() - try: - size_statistics += sync_repo(remote_url, - local_dir, Path(tmpdir), args.delete) - except Exception: - logging.exception("Failed to sync repo: {}".format(repo)) - finally: - shutil.rmtree(tmpdir) + tmpdir = tempfile.mkdtemp() + try: + size_statistics += sync_repo(remote_url, local_dir, + Path(tmpdir), args.delete) + except Exception: + logging.exception("Failed to sync repo: {}".format(repo)) + finally: + shutil.rmtree(tmpdir) print("Total size is", sizeof_fmt(size_statistics, suffix="")) + if __name__ == "__main__": main()