From 9eff258f33c6726fde91def103cbba05edf55a6e Mon Sep 17 00:00:00 2001 From: dramforever Date: Fri, 1 May 2020 18:45:20 +0800 Subject: [PATCH] nix-channel: dedup paths before download Could fix a race that causes FileNotFoundError --- nix-channels.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/nix-channels.py b/nix-channels.py index b453319..115e11b 100755 --- a/nix-channels.py +++ b/nix-channels.py @@ -290,7 +290,8 @@ def update_channels(channels): logging.info(f' - {len(paths)} paths listed') - todo = {} + todo = [] + seen_paths = set() channel_failure = False # Batch paths to avoid E2BIG @@ -313,7 +314,14 @@ def update_channels(channels): infos = json.loads(process.stdout) for info in infos: ha = hash_part(info['path']) - todo[ha] = (info['url'], f'{ha}.narinfo') + one_todo = [ + name + for name in [info['url'], f'{ha}.narinfo'] + if name not in seen_paths + ] + seen_paths.update(one_todo) + if one_todo: + todo.append(one_todo) else: logging.info(f' - {len(todo)} paths to download') @@ -336,7 +344,7 @@ def update_channels(channels): with ThreadPoolExecutor(max_workers=THREADS) as executor: results = executor.map( lambda job: try_mirror(*job), - enumerate(todo.values()) + enumerate(todo) ) if not all(results): channel_failure = True