yacloud/tunasync-scripts
1
0
Fork 0
mirror of https://github.com/tuna/tunasync-scripts.git synced 2025-04-20 04:12:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

nix-channel: dedup paths before download

Browse Source 
Could fix a race that causes FileNotFoundError
This commit is contained in:
dramforever 2020-05-01 18:45:20 +08:00
parent 86950766c9
commit 9eff258f33
1 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
nix-channels.py
View File

@ -290,7 +290,8 @@ def update_channels(channels):
logging.info(f' - {len(paths)} paths listed') logging.info(f' - {len(paths)} paths listed')
todo = {} todo = []
seen_paths = set()
channel_failure = False channel_failure = False
# Batch paths to avoid E2BIG # Batch paths to avoid E2BIG
@ -313,7 +314,14 @@ def update_channels(channels):
infos = json.loads(process.stdout) infos = json.loads(process.stdout)
for info in infos: for info in infos:
ha = hash_part(info['path']) ha = hash_part(info['path'])
todo[ha] = (info['url'], f'{ha}.narinfo') one_todo = [
name
for name in [info['url'], f'{ha}.narinfo']
if name not in seen_paths
]
seen_paths.update(one_todo)
if one_todo:
todo.append(one_todo)
else: else:
logging.info(f' - {len(todo)} paths to download') logging.info(f' - {len(todo)} paths to download')
@ -336,7 +344,7 @@ def update_channels(channels):
with ThreadPoolExecutor(max_workers=THREADS) as executor: with ThreadPoolExecutor(max_workers=THREADS) as executor:
results = executor.map( results = executor.map(
lambda job: try_mirror(*job), lambda job: try_mirror(*job),
enumerate(todo.values()) enumerate(todo)
) )
if not all(results): if not all(results):
channel_failure = True channel_failure = True