yacloud/shadowmire
1
0
Fork 0
mirror of https://github.com/taoky/shadowmire.git synced 2025-07-08 09:12:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Formatting and linting

Browse Source
This commit is contained in:
taoky 2024-08-02 16:01:59 +08:00
parent 18a3ebbe03
commit b0187e947f
1 changed files with 27 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
shadowmire.py
View File

@ -9,11 +9,11 @@ from pathlib import Path
from html.parser import HTMLParser from html.parser import HTMLParser
import logging import logging
import html import html
import requests
import argparse import argparse
import os import os
from contextlib import contextmanager from contextlib import contextmanager
import sqlite3 import sqlite3
import requests
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -27,6 +27,7 @@ class LocalVersionKV:
As it would have consistency issue if it's writing while downstream is downloading the database. As it would have consistency issue if it's writing while downstream is downloading the database.
An extra "jsonpath" is used, to store kv results when necessary. An extra "jsonpath" is used, to store kv results when necessary.
""" """
def __init__(self, dbpath: Path, jsonpath: Path) -> None: def __init__(self, dbpath: Path, jsonpath: Path) -> None:
self.conn = sqlite3.connect(dbpath) self.conn = sqlite3.connect(dbpath)
self.jsonpath = jsonpath self.jsonpath = jsonpath
@ -66,6 +67,12 @@ class LocalVersionKV:
if commit: if commit:
self.conn.commit() self.conn.commit()
def keys(self) -> list[str]:
cur = self.conn.cursor()
res = cur.execute("SELECT key FROM local")
rows = res.fetchall()
return [row[0] for row in rows]
def dump(self) -> dict[str, int]: def dump(self) -> dict[str, int]:
cur = self.conn.cursor() cur = self.conn.cursor()
res = cur.execute("SELECT key, value FROM local") res = cur.execute("SELECT key, value FROM local")
@ -294,7 +301,9 @@ class Plan:
class SyncBase: class SyncBase:
def __init__(self, basedir: Path, local_db: LocalVersionKV, sync_packages: bool = False) -> None: def __init__(
self, basedir: Path, local_db: LocalVersionKV, sync_packages: bool = False
) -> None:
self.basedir = basedir self.basedir = basedir
self.local_db = local_db self.local_db = local_db
self.simple_dir = basedir / "simple" self.simple_dir = basedir / "simple"
@ -364,7 +373,7 @@ class SyncBase:
raise NotImplementedError raise NotImplementedError
def finalize(self) -> None: def finalize(self) -> None:
assert self.remote local_names = self.local_db.keys()
# generate index.html at basedir # generate index.html at basedir
index_path = self.basedir / "simple" / "index.html" index_path = self.basedir / "simple" / "index.html"
# modified from bandersnatch # modified from bandersnatch
@ -378,17 +387,16 @@ class SyncBase:
f.write(" <body>\n") f.write(" <body>\n")
# This will either be the simple dir, or if we are using index # This will either be the simple dir, or if we are using index
# directory hashing, a list of subdirs to process. # directory hashing, a list of subdirs to process.
for pkg in self.remote: for pkg in local_names:
# We're really trusty that this is all encoded in UTF-8. :/ # We're really trusty that this is all encoded in UTF-8. :/
f.write(f' <a href="{pkg}/">{pkg}</a><br/>\n') f.write(f' <a href="{pkg}/">{pkg}</a><br/>\n')
f.write(" </body>\n</html>") f.write(" </body>\n</html>")
remote_json_path = self.basedir / "remote.json"
local_json_path = self.basedir / "local.json"
remote_json_path.rename(local_json_path)
class SyncPyPI(SyncBase): class SyncPyPI(SyncBase):
def __init__(self, basedir: Path, local_db: LocalVersionKV, sync_packages: bool = False) -> None: def __init__(
self, basedir: Path, local_db: LocalVersionKV, sync_packages: bool = False
) -> None:
self.pypi = PyPI() self.pypi = PyPI()
self.session = create_requests_session() self.session = create_requests_session()
super().__init__(basedir, local_db, sync_packages) super().__init__(basedir, local_db, sync_packages)
@ -410,7 +418,7 @@ class SyncPyPI(SyncBase):
logger.warning("%s missing from upstream, skip.", package_name) logger.warning("%s missing from upstream, skip.", package_name)
return False return False
last_serial = meta['last_serial'] last_serial = meta["last_serial"]
# OK, here we don't bother store raw name # OK, here we don't bother store raw name
# Considering that JSON API even does not give package raw name, why bother we use it? # Considering that JSON API even does not give package raw name, why bother we use it?
simple_html_contents = self.pypi.generate_html_simple_page(meta, package_name) simple_html_contents = self.pypi.generate_html_simple_page(meta, package_name)
@ -435,7 +443,11 @@ class SyncPyPI(SyncBase):
class SyncPlainHTTP(SyncBase): class SyncPlainHTTP(SyncBase):
def __init__( def __init__(
self, upstream: str, basedir: Path, local_db: LocalVersionKV, sync_packages: bool = False self,
upstream: str,
basedir: Path,
local_db: LocalVersionKV,
sync_packages: bool = False,
) -> None: ) -> None:
self.upstream = upstream self.upstream = upstream
self.session = create_requests_session() self.session = create_requests_session()
@ -448,7 +460,7 @@ class SyncPlainHTTP(SyncBase):
remote: dict[str, int] = resp.json() remote: dict[str, int] = resp.json()
return remote return remote
def do_update(self, package_name) -> bool: def do_update(self, package_name: str) -> bool:
package_simple_path = self.simple_dir / package_name package_simple_path = self.simple_dir / package_name
package_simple_path.mkdir(exist_ok=True) package_simple_path.mkdir(exist_ok=True)
# directly fetch remote files # directly fetch remote files
@ -461,8 +473,7 @@ class SyncPlainHTTP(SyncBase):
continue continue
else: else:
logger.error("%s does not exist. Stop with this.", file_url) logger.error("%s does not exist. Stop with this.", file_url)
# TODO: error handling return False
break
else: else:
resp.raise_for_status() resp.raise_for_status()
content = resp.content content = resp.content