diff --git a/README.md b/README.md index 3241b3a..8e7fd31 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Shadowmire syncs PyPI (or plain HTTP(S) PyPI mirrors using Shadowmire) with a lightweight and easy approach. +Requires Python 3.11+. + ## Docs ### Background @@ -89,6 +91,12 @@ Verify command could be used if you believe that something is wrong (inconsisten Verify command accepts same arguments as sync. +If you don't like appending a long argument list, you could use `--config` ([example](./config.example.toml)): + +```shell +./shadowmire.py --config config.toml sync +``` + Also, if you need debugging, you could use `do-update` and `do-remove` command to operate on a single package. ## Acknowledgements diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 0000000..00adf0e --- /dev/null +++ b/config.example.toml @@ -0,0 +1,9 @@ +[options] +sync_packages = true +# shadowmire_upstream = http://example.com/pypi/web/ +exclude = [ + "[a-z]" +] +prerelease_exclude = [ + ".+" +] diff --git a/shadowmire.py b/shadowmire.py index 82f7a09..8d12a39 100755 --- a/shadowmire.py +++ b/shadowmire.py @@ -15,6 +15,7 @@ from contextlib import contextmanager import sqlite3 from concurrent.futures import ThreadPoolExecutor, as_completed import signal +import tomllib import requests import click from tqdm import tqdm @@ -481,7 +482,7 @@ class SyncBase: logger.warning( "%s generated an exception", package_name, exc_info=True ) - if idx % 1000 == 0: + if idx % 100 == 0: self.local_db.dump_json() except (ExitProgramException, KeyboardInterrupt): logger.info("Get ExitProgramException or KeyboardInterrupt, exiting...") @@ -704,8 +705,8 @@ class SyncPlainHTTP(SyncBase): package_simple_path = self.simple_dir / package_name package_simple_path.mkdir(exist_ok=True) if self.sync_packages: - existing_hrefs = get_existing_hrefs(package_simple_path) - existing_hrefs = [] if existing_hrefs is None else existing_hrefs + hrefs = get_existing_hrefs(package_simple_path) + existing_hrefs = [] if hrefs is None else hrefs # Download JSON meta file_url = urljoin(self.upstream, f"/json/{package_name}") success, resp = download( @@ -781,9 +782,9 @@ def get_local_serial(package_meta_path: Path) -> Optional[int]: def sync_shared_args(func): shared_options = [ click.option( - "--sync-packages", - is_flag=True, - help="Sync packages instead of just indexes", + "--sync-packages/--no-sync-packages", + default=False, + help="Sync packages instead of just indexes, by default it's --no-sync-packages", ), click.option( "--shadowmire-upstream", @@ -805,7 +806,33 @@ def sync_shared_args(func): return func +def read_config( + ctx: click.Context, param: click.Option, filename: Optional[str] +) -> None: + if filename is None: + return + with open(filename, "rb") as f: + data = tomllib.load(f) + try: + options = dict(data["options"]) + except KeyError: + options = {} + ctx.default_map = { + "sync": options, + "verify": options, + "do-update": options, + "do-remove": options, + } + + @click.group() +@click.option( + "--config", + type=click.Path(dir_okay=False), + help="Read option defaults from specified TOML file", + callback=read_config, + expose_value=False, +) @click.pass_context def cli(ctx: click.Context) -> None: log_level = logging.DEBUG if os.environ.get("DEBUG") else logging.INFO @@ -869,7 +896,7 @@ def sync( plan = syncer.determine_sync_plan(local, excludes) # save plan for debugging with overwrite(basedir / "plan.json") as f: - json.dump(plan, f, default=vars) + json.dump(plan, f, default=vars, indent=2) syncer.do_sync_plan(plan, prerelease_excludes) syncer.finalize() @@ -922,8 +949,10 @@ def verify( for package_name in plan.remove: # We only take the plan.remove part here syncer.do_remove(package_name) - - logger.info("make sure all local indexes are valid, and (if --sync-packages) have valid local package files") + + logger.info( + "make sure all local indexes are valid, and (if --sync-packages) have valid local package files" + ) syncer.check_and_update(list(local_names)) syncer.finalize()