From 7d1b156b33985478f8604bbdaf711a5334e52741 Mon Sep 17 00:00:00 2001 From: Yuxiang Zhang Date: Sat, 1 Feb 2020 15:55:25 +0800 Subject: [PATCH] skip package files only --- docker-ce.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/docker-ce.py b/docker-ce.py index 8018569..2326081 100755 --- a/docker-ce.py +++ b/docker-ce.py @@ -98,7 +98,23 @@ def downloading_worker(q): if item is None: break - url, dst_file = item + url, dst_file, working_dir = item + if dst_file.is_file(): + print("checking", url, flush=True) + r = requests.head(url, timeout=TIMEOUT_OPTION) + remote_filesize = int(r.headers['content-length']) + remote_date = parsedate_to_datetime(r.headers['last-modified']) + stat = dst_file.stat() + local_filesize = stat.st_size + local_mtime = stat.st_mtime + + if remote_filesize == local_filesize and remote_date.timestamp() == local_mtime: + print("skipping", dst_file.relative_to(working_dir), flush=True) + q.task_done() + continue + + dst_file.unlink() + print("downloading", url, flush=True) try: requests_download(url, dst_file) @@ -106,6 +122,7 @@ def downloading_worker(q): print("Failed to download", url, flush=True) if dst_file.is_file(): dst_file.unlink() + q.task_done() @@ -125,7 +142,7 @@ def main(): parser.add_argument("--workers", default=1, type=int, help='number of concurrent downloading jobs') parser.add_argument("--fast-skip", action='store_true', - help='do not verify size and timestamp of existing files') + help='do not verify size and timestamp of existing package files') args = parser.parse_args() if args.working_dir is None: @@ -141,26 +158,13 @@ def main(): remote_filelist.append(dst_file.relative_to(working_dir)) if dst_file.is_file(): - if args.fast_skip: - print("Skipping", dst_file.relative_to(working_dir), flush=True) + if args.fast_skip and dst_file.suffix in ['.rpm', '.deb', '.tgz', '.zip']: + print("fast skipping", dst_file.relative_to(working_dir), flush=True) continue - - r = requests.head(url, timeout=TIMEOUT_OPTION) - remote_filesize = int(r.headers['content-length']) - remote_date = parsedate_to_datetime(r.headers['last-modified']) - stat = dst_file.stat() - local_filesize = stat.st_size - local_mtime = stat.st_mtime - - if remote_filesize == local_filesize and remote_date.timestamp() == local_mtime: - print("Skipping", dst_file.relative_to(working_dir), flush=True) - continue - - dst_file.unlink() else: dst_file.parent.mkdir(parents=True, exist_ok=True) - task_queue.put((url, dst_file)) + task_queue.put((url, dst_file, working_dir)) # block until all tasks are done task_queue.join()