diff --git a/nix-channels.py b/nix-channels.py index bec8ec9..0050e53 100644 --- a/nix-channels.py +++ b/nix-channels.py @@ -258,7 +258,56 @@ def clone_channels(): return channels_to_update +def make_xargs(): + arg_max = int(subprocess.run( + [ 'getconf', 'ARG_MAX' ], + stdout=subprocess.PIPE + ).stdout) + + env_size = sum( + len(k.encode()) + len(v.encode()) + 2 + for k, v in os.environ.items() + ) + + # Subtract 1024 to be safe safe + usable = arg_max - env_size - 1024 + + logging.info(f'- mk_xargs: Usable command line bytes: {usable}') + + def check_run(args): + global failure + p = subprocess.run(args) + if p.returncode != 0: + logging.info(f' - Subprocess failed with code {p.returncode}') + failure = True + + def xargs(common, args): + common = [ a.encode() if isinstance(a, str) else a for a in common ] + args = [ a.encode() if isinstance(a, str) else a for a in args ] + + common_size = sum(len(a) + 1 for a in common) + + buf, cur_size = [], common_size + + for a in args: + size = len(a) + 1 + if cur_size + size > usable: + check_run(common + buf) + buf, cur_size = [], common_size + + buf.append(a) + cur_size += size + + assert cur_size <= usable, f'Argument {a[:50]}... will never fit' + + if buf: + check_run(common + buf) + + return xargs + def update_channels(channels): + xargs = make_xargs() + logging.info(f'- Updating binary cache') has_cache_info = False @@ -287,31 +336,17 @@ def update_channels(channels): logging.info(f' - {len(paths)} paths listed') - # xargs can splits up the argument lists and invokes nix copy multiple - # times to avoid E2BIG (Argument list too long) - nix_process = subprocess.Popen( - [ 'xargs', 'nix', 'copy', + xargs( + [ + 'nix', 'copy', '--from', upstream_binary_cache, '--to', nix_store_dest, '--verbose' ], - universal_newlines=True, - stdin=subprocess.PIPE + paths ) - for path in paths: - nix_process.stdin.write(path.decode() + '\n') - - retcode = nix_process.wait() - - if retcode == 0: - chan_path_update.rename(working_dir / channel) - logging.info(f' - Done') - else: - global failure - failure = True - - logging.info(f' - nix copy failed') + logging.info(f' - Finished') if __name__ == '__main__': channels = clone_channels()