yacloud/tunasync-scripts
1
0
Fork 0
mirror of https://github.com/tuna/tunasync-scripts.git synced 2025-04-19 11:42:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #178 from taoky/anaconda-1

Browse Source 
anaconda: add repodata zst and remove legacy support
This commit is contained in:
Shengqi Chen 2024-08-17 10:02:48 +08:00 committed by GitHub
commit b9697ced2a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
anaconda.py
View File

@ -124,22 +124,29 @@ def curl_download(remote_url: str, dst_file: Path, sha256: str = None, md5: str
return "MD5 mismatch" return "MD5 mismatch"
def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool): def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool, remove_legacy: bool):
logging.info("Start syncing {}".format(repo_url)) logging.info("Start syncing {}".format(repo_url))
local_dir.mkdir(parents=True, exist_ok=True) local_dir.mkdir(parents=True, exist_ok=True)
repodata_url = repo_url + '/repodata.json' repodata_url = repo_url + '/repodata.json'
bz2_repodata_url = repo_url + '/repodata.json.bz2' bz2_repodata_url = repo_url + '/repodata.json.bz2'
# https://github.com/conda/conda/issues/13256, from conda 24.1.x
zst_repodata_url = repo_url + '/repodata.json.zst'
# https://docs.conda.io/projects/conda-build/en/latest/release-notes.html # https://docs.conda.io/projects/conda-build/en/latest/release-notes.html
# "current_repodata.json" - like repodata.json, but only has the newest version of each file # "current_repodata.json" - like repodata.json, but only has the newest version of each file
current_repodata_url = repo_url + '/current_repodata.json' current_repodata_url = repo_url + '/current_repodata.json'
tmp_repodata = tmpdir / "repodata.json" tmp_repodata = tmpdir / "repodata.json"
tmp_bz2_repodata = tmpdir / "repodata.json.bz2" tmp_bz2_repodata = tmpdir / "repodata.json.bz2"
tmp_zst_repodata = tmpdir / "repodata.json.zst"
tmp_current_repodata = tmpdir / 'current_repodata.json' tmp_current_repodata = tmpdir / 'current_repodata.json'
curl_download(repodata_url, tmp_repodata) curl_download(repodata_url, tmp_repodata)
curl_download(bz2_repodata_url, tmp_bz2_repodata) curl_download(bz2_repodata_url, tmp_bz2_repodata)
try:
curl_download(zst_repodata_url, tmp_zst_repodata)
except:
pass
try: try:
curl_download(current_repodata_url, tmp_current_repodata) curl_download(current_repodata_url, tmp_current_repodata)
except: except:
@ -150,9 +157,14 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool):
remote_filelist = [] remote_filelist = []
total_size = 0 total_size = 0
packages = repodata['packages'] legacy_packages = repodata['packages']
if 'packages.conda' in repodata: conda_packages = repodata.get("packages.conda", {})
packages.update(repodata['packages.conda']) if remove_legacy:
# https://github.com/anaconda/conda/blob/0dbf85e0546e0b0dc060c8265ec936591ccbe980/conda/core/subdir_data.py#L440-L442
use_legacy_packages = set(legacy_packages.keys()) - set(k[:-6] + ".tar.bz2" for k in conda_packages.keys())
legacy_packages = {k: legacy_packages[k] for k in use_legacy_packages}
packages = {**legacy_packages, **conda_packages}
for filename, meta in packages.items(): for filename, meta in packages.items():
if meta['name'] in EXCLUDED_PACKAGES: if meta['name'] in EXCLUDED_PACKAGES:
continue continue
@ -207,6 +219,10 @@ def sync_repo(repo_url: str, local_dir: Path, tmpdir: Path, delete: bool):
shutil.move(str(tmp_repodata), str(local_dir / "repodata.json")) shutil.move(str(tmp_repodata), str(local_dir / "repodata.json"))
shutil.move(str(tmp_bz2_repodata), str(local_dir / "repodata.json.bz2")) shutil.move(str(tmp_bz2_repodata), str(local_dir / "repodata.json.bz2"))
try:
shutil.move(str(tmp_zst_repodata), str(local_dir / "repodata.json.zst"))
except:
pass
tmp_current_repodata_gz_gened = False tmp_current_repodata_gz_gened = False
if tmp_current_repodata.is_file(): if tmp_current_repodata.is_file():
if os.path.getsize(tmp_current_repodata) > GEN_METADATA_JSON_GZIP_THRESHOLD: if os.path.getsize(tmp_current_repodata) > GEN_METADATA_JSON_GZIP_THRESHOLD:
@ -307,6 +323,8 @@ def main():
parser.add_argument("--working-dir", default=WORKING_DIR) parser.add_argument("--working-dir", default=WORKING_DIR)
parser.add_argument("--delete", action='store_true', parser.add_argument("--delete", action='store_true',
help='delete unreferenced package files') help='delete unreferenced package files')
parser.add_argument("--remove-legacy", action='store_true',
help='delete legacy packages which have conda counterpart. Requires client conda >= 4.7.0')
args = parser.parse_args() args = parser.parse_args()
if args.working_dir is None: if args.working_dir is None:
@ -335,7 +353,7 @@ def main():
tmpdir = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp()
try: try:
size_statistics += sync_repo(remote_url, size_statistics += sync_repo(remote_url,
local_dir, Path(tmpdir), args.delete) local_dir, Path(tmpdir), args.delete, args.remove_legacy)
except Exception: except Exception:
logging.exception("Failed to sync repo: {}/{}".format(repo, arch)) logging.exception("Failed to sync repo: {}/{}".format(repo, arch))
finally: finally:
@ -348,7 +366,7 @@ def main():
tmpdir = tempfile.mkdtemp() tmpdir = tempfile.mkdtemp()
try: try:
size_statistics += sync_repo(remote_url, size_statistics += sync_repo(remote_url,
local_dir, Path(tmpdir), args.delete) local_dir, Path(tmpdir), args.delete, args.remove_legacy)
except Exception: except Exception:
logging.exception("Failed to sync repo: {}".format(repo)) logging.exception("Failed to sync repo: {}".format(repo))
finally: finally: