nix-channels: Split argument list in script

nix-channels.py

@@ -258,7 +258,56 @@ def clone_channels():
 
     return channels_to_update
 
+def make_xargs():
+    arg_max = int(subprocess.run(
+        [ 'getconf', 'ARG_MAX' ],
+        stdout=subprocess.PIPE
+    ).stdout)
+
+    env_size = sum(
+        len(k.encode()) + len(v.encode()) + 2
+        for k, v in os.environ.items()
+    )
+
+    # Subtract 1024 to be safe safe
+    usable = arg_max - env_size - 1024
+
+    logging.info(f'- mk_xargs: Usable command line bytes: {usable}')
+
+    def check_run(args):
+        global failure
+        p = subprocess.run(args)
+        if p.returncode != 0:
+            logging.info(f'    - Subprocess failed with code {p.returncode}')
+            failure = True
+
+    def xargs(common, args):
+        common = [ a.encode() if isinstance(a, str) else a for a in common ]
+        args = [ a.encode() if isinstance(a, str) else a for a in args ]
+
+        common_size = sum(len(a) + 1 for a in common)
+
+        buf, cur_size = [], common_size
+
+        for a in args:
+            size = len(a) + 1
+            if cur_size + size > usable:
+                check_run(common + buf)
+                buf, cur_size = [], common_size
+
+            buf.append(a)
+            cur_size += size
+
+            assert cur_size <= usable, f'Argument {a[:50]}... will never fit'
+
+        if buf:
+            check_run(common + buf)
+
+    return xargs
+
 def update_channels(channels):
+    xargs = make_xargs()
+
     logging.info(f'- Updating binary cache')
 
     has_cache_info = False
@@ -287,31 +336,17 @@ def update_channels(channels):
 
         logging.info(f'    - {len(paths)} paths listed')
 
-        # xargs can splits up the argument lists and invokes nix copy multiple
-        # times to avoid E2BIG (Argument list too long)
-        nix_process = subprocess.Popen(
-            [ 'xargs', 'nix', 'copy',
+        xargs(
+            [
+                'nix', 'copy',
                 '--from', upstream_binary_cache,
                 '--to', nix_store_dest,
                 '--verbose'
             ],
-            universal_newlines=True,
-            stdin=subprocess.PIPE
+            paths
         )
 
-        for path in paths:
-            nix_process.stdin.write(path.decode() + '\n')
-
-        retcode = nix_process.wait()
-
-        if retcode == 0:
-            chan_path_update.rename(working_dir / channel)
-            logging.info(f'    - Done')
-        else:
-            global failure
-            failure = True
-
-            logging.info(f'    - nix copy failed')
+        logging.info(f'    - Finished')
 
 if __name__ == '__main__':
     channels = clone_channels()