yacloud/tunasync-scripts
1
0
Fork 0
mirror of https://github.com/tuna/tunasync-scripts.git synced 2025-04-20 04:12:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

github-raw: add rosdep

Browse Source 
Ref to https://github.com/tuna/issues/issues/1204
This commit is contained in:
Zenithal 2021-11-18 20:51:43 +08:00
parent 0775ccadbc
commit 942fe00303
No known key found for this signature in database
GPG Key ID: 1189C659F3D04C1C
1 changed files with 28 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
github-raw.py
View File

@ -11,6 +11,9 @@ import hashlib
import requests import requests
def raw_to_tuna(s: str) -> str:
return s.replace("https://raw.githubusercontent.com/",
"https://mirrors.tuna.tsinghua.edu.cn/github-raw/")
BASE_URL = os.getenv("TUNASYNC_UPSTREAM_URL", "https://api.github.com/repos/") BASE_URL = os.getenv("TUNASYNC_UPSTREAM_URL", "https://api.github.com/repos/")
WORKING_DIR = os.getenv("TUNASYNC_WORKING_DIR") WORKING_DIR = os.getenv("TUNASYNC_WORKING_DIR")
@ -19,6 +22,13 @@ REPOS = [
## for stackage ## for stackage
["fpco/minghc", "master", "bin", "7z.exe"], ["fpco/minghc", "master", "bin", "7z.exe"],
["fpco/minghc", "master", "bin", "7z.dll"], ["fpco/minghc", "master", "bin", "7z.dll"],
## for rosdep
{ "path": ["ros/rosdistro", "master", "rosdep", "sources.list.d", "20-default.list"], "filter": raw_to_tuna },
["ros/rosdistro", "master", "rosdep", "osx-homebrew.yaml"],
["ros/rosdistro", "master", "rosdep", "base.yaml"],
["ros/rosdistro", "master", "rosdep", "python.yaml"],
["ros/rosdistro", "master", "rosdep", "ruby.yaml"],
["ros/rosdistro", "master", "releases", "fuerte.yaml"],
] ]
# connect and read timeout value # connect and read timeout value
@ -48,7 +58,7 @@ def github_blob(*args, **kwargs):
kwargs['headers'] = headers kwargs['headers'] = headers
return github_get(*args, **kwargs) return github_get(*args, **kwargs)
def do_download(remote_url: str, dst_file: Path, remote_size: int, sha: str): def do_download(remote_url: str, dst_file: Path, remote_size: int, sha: str, filter=None):
# NOTE the stream=True parameter below # NOTE the stream=True parameter below
with github_blob(remote_url, stream=True) as r: with github_blob(remote_url, stream=True) as r:
r.raise_for_status() r.raise_for_status()
@ -64,6 +74,12 @@ def do_download(remote_url: str, dst_file: Path, remote_size: int, sha: str):
downloaded_size = tmp_dst_file.stat().st_size downloaded_size = tmp_dst_file.stat().st_size
if remote_size != -1 and downloaded_size != remote_size: if remote_size != -1 and downloaded_size != remote_size:
raise Exception(f'File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes') raise Exception(f'File {dst_file.as_posix()} size mismatch: downloaded {downloaded_size} bytes, expected {remote_size} bytes')
if filter != None:
with open(tmp_dst_file, "r+") as f:
s = filter(f.read())
f.seek(0)
f.truncate()
f.write(s)
tmp_dst_file.chmod(0o644) tmp_dst_file.chmod(0o644)
target = dst_file.parent / ".sha" / sha target = dst_file.parent / ".sha" / sha
print("symlink", dst_file) print("symlink", dst_file)
@ -87,6 +103,8 @@ def downloading_worker(q):
if item is None: if item is None:
break break
filter = item.pop(0) # remove filter
dst_file = Path('/'.join(item)) dst_file = Path('/'.join(item))
dst_file.parent.mkdir(parents=True, exist_ok=True) dst_file.parent.mkdir(parents=True, exist_ok=True)
@ -119,9 +137,8 @@ def downloading_worker(q):
else: else:
raise Exception raise Exception
if not dst_file.is_symlink() or \ if not dst_file.is_symlink() or \
dst_file.stat().st_size != size or \
Path(os.readlink(dst_file)).name != sha: Path(os.readlink(dst_file)).name != sha:
do_download(url, dst_file, size, sha) do_download(url, dst_file, size, sha, filter)
else: else:
print("Skip", dst_file) print("Skip", dst_file)
except Exception as e: except Exception as e:
@ -155,8 +172,14 @@ def main():
task_queue = create_workers(args.workers) task_queue = create_workers(args.workers)
for cfg in REPOS: for cfg in REPOS:
if isinstance(cfg, list):
cfg.insert(0, working_dir) cfg.insert(0, working_dir)
cfg.insert(0, None)
task_queue.put(cfg) task_queue.put(cfg)
else:
cfg["path"].insert(0, working_dir)
cfg["path"].insert(0, cfg["filter"])
task_queue.put(cfg["path"])
# block until all tasks are done # block until all tasks are done
task_queue.join() task_queue.join()