yacloud/shadowmire
1
0
Fork 0
mirror of https://github.com/taoky/shadowmire.git synced 2025-07-08 09:12:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add multithreading to genlocal

Browse Source 
Related: #4
This commit is contained in:
taoky 2024-09-04 14:08:18 +08:00
parent 20c1462ec7
commit 99bd4d932e
1 changed files with 24 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
shadowmire.py
View File

@ -1158,11 +1158,30 @@ def genlocal(ctx: click.Context) -> None:
logger.info("Iterating all items under %s", json_dir) logger.info("Iterating all items under %s", json_dir)
dir_items = [d for d in fast_iterdir(json_dir, "file")] dir_items = [d for d in fast_iterdir(json_dir, "file")]
logger.info("Detected %s packages in %s in total", len(dir_items), json_dir) logger.info("Detected %s packages in %s in total", len(dir_items), json_dir)
for package_metapath in tqdm(dir_items, desc="Reading packages from json/"): with ThreadPoolExecutor(max_workers=IOWORKERS) as executor:
package_name = package_metapath.name futures = {
serial = get_local_serial(package_metapath) executor.submit(get_local_serial, package_metapath): package_metapath
if serial: for package_metapath in dir_items
local[package_name] = serial }
try:
for future in tqdm(
as_completed(futures),
total=len(dir_items),
desc="Reading packages from json/",
):
package_name = futures[future].name
try:
serial = future.result()
if serial:
local[package_name] = serial
except Exception as e:
if isinstance(e, (KeyboardInterrupt)):
raise
logger.warning(
"%s generated an exception", package_name, exc_info=True
)
except (ExitProgramException, KeyboardInterrupt):
exit_with_futures(futures)
logger.info( logger.info(
"%d out of %d packages have valid serial number", len(local), len(dir_items) "%d out of %d packages have valid serial number", len(local), len(dir_items)
) )