From 3f84a93398683b3ac352929a5d3aefcecdbc4768 Mon Sep 17 00:00:00 2001 From: Wang Yuzhi Date: Mon, 23 Jan 2017 20:11:15 +0800 Subject: [PATCH 01/21] added 'noarch' to anaconda.sh --- anaconda.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anaconda.sh b/anaconda.sh index d6429dc..17c500c 100755 --- a/anaconda.sh +++ b/anaconda.sh @@ -18,10 +18,10 @@ LOCAL_DIR_BASE="${TUNASYNC_WORKING_DIR}" TMP_DIR=$(mktemp -d) CONDA_REPOS=("free" "r" "mro" "pro") -CONDA_ARCHES=("linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32") +CONDA_ARCHES=("noarch" "linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32") CONDA_CLOUD_REPOS=( - "conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64" + "conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64" "conda-forge/noarch" "msys2/win-64" "r/linux-64" "r/osx-64" "r/osx-64" ) From 9cfdda0b6b9aa49062cafffc2a01b7ce99cda3c5 Mon Sep 17 00:00:00 2001 From: Wang Yuzhi Date: Mon, 23 Jan 2017 20:14:42 +0800 Subject: [PATCH 02/21] remove conda cloud r channel --- anaconda.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/anaconda.sh b/anaconda.sh index 17c500c..dc8cac1 100755 --- a/anaconda.sh +++ b/anaconda.sh @@ -20,10 +20,7 @@ TMP_DIR=$(mktemp -d) CONDA_REPOS=("free" "r" "mro" "pro") CONDA_ARCHES=("noarch" "linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32") -CONDA_CLOUD_REPOS=( - "conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64" "conda-forge/noarch" - "msys2/win-64" "r/linux-64" "r/osx-64" "r/osx-64" -) +CONDA_CLOUD_REPOS=("conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64" "conda-forge/noarch" "msys2/win-64") EXIT_STATUS=0 EXIT_MSG="" From 60e7aca5613c93d7db8ebc5d5dbaed0a23c3ee82 Mon Sep 17 00:00:00 2001 From: Shao Cheng Date: Mon, 23 Jan 2017 20:25:04 +0800 Subject: [PATCH 03/21] Add stackage sync script. --- Dockerfile | 4 ++- stackage.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 stackage.py diff --git a/Dockerfile b/Dockerfile index 1c684de..45b8352 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,10 +7,12 @@ RUN echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie main contrib no echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security/ jessie/updates main contrib non-free" >> /etc/apt/sources.list RUN apt-get update && \ - apt-get install -y wget curl rsync lftp git jq python-dev python-pip yum-utils createrepo python3-dev python3-pip + apt-get install -y wget curl rsync lftp git jq python-dev python-pip yum-utils createrepo python3-dev python3-pip aria2 RUN pip install --upgrade pip setuptools && \ pip install bandersnatch==1.11 + +RUN pip3 install requests pyyaml RUN mkdir -p /home/tunasync-scripts ADD https://storage.googleapis.com/git-repo-downloads/repo /usr/local/bin/aosp-repo diff --git a/stackage.py b/stackage.py new file mode 100644 index 0000000..76d0f73 --- /dev/null +++ b/stackage.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +# python deps: requests, pyyaml +# non-python deps: aria2, git + +import os +import pathlib +import requests +import subprocess +import yaml + + +class StackageSession(object): + def __init__(self): + self._base_path = pathlib.Path(os.environ['TUNASYNC_WORKING_DIR']) + + def download(self, dir_, url, sha1=None, force=False): + dir_path = self._base_path / dir_ + file_path = dir_path / url.split('/')[-1] + if force and file_path.is_file(): + file_path.unlink() + if file_path.is_file(): + print('{} exists, skipping'.format(file_path)) + else: + args = ['aria2c', url, '--dir={}'.format(dir_path)] + if sha1: + args.append('--checksum=sha-1={}'.format(sha1)) + if subprocess.run(args).returncode != 0 and file_path.is_file(): + file_path.unlink() + raise OSError('Download failed for {}'.format(url)) + else: + print('Downloaded {} to {}'.format(url, file_path)) + + def load_stack_setup(self): + d = yaml.load(requests.get( + 'https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml').content) + for platform in d['ghc']: + for ver in d['ghc'][platform]: + self.download('ghc', d['ghc'][platform][ver]['url'], d['ghc'][platform][ver]['sha1']) + d['ghc'][platform][ver]['url'] = 'http://mirrors.tuna.tsinghua.edu.cn/stackage/ghc/{}'.format( + d['ghc'][platform][ver]['url'].split('/')[-1]) + d['msys2'] = {'windows32': {'version': '20161025', + 'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/i686/msys2-base-i686-20161025.tar.xz', + 'content-length': 47526500, + 'sha1': '5D17FA53077A93A38A9AC0ACB8A03BF6C2FC32AD'}, + 'windows64': {'version': '20161025', + 'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz', + 'content-length': 47166584, + 'sha1': '05FD74A6C61923837DFFE22601C9014F422B5460'}} + with open(self._base_path / 'stack-setup.yaml', 'w') as f: + yaml.dump(d, f) + print('Loaded stack-setup.yaml') + + def load_stackage_snapshots(self): + for channel in ['lts-haskell', 'stackage-nightly']: + if (self._base_path / channel).is_dir(): + args = ['git', '-C', self._base_path / channel, 'pull'] + else: + args = ['git', '-C', self._base_path, 'clone', '--depth', '1', + 'https://github.com/fpco/{}.git'.format(channel)] + subprocess.run(args, check=True) + print('Loaded {}'.format(channel)) + self.download('', 'https://www.stackage.org/download/snapshots.json', force=True) + print('Loaded snapshots.json') + + +if __name__ == '__main__': + s = StackageSession() + s.load_stackage_snapshots() + s.load_stack_setup() From d70240b96cc61f24f54f76c9792c83ca9258e3bd Mon Sep 17 00:00:00 2001 From: bigeagle Date: Mon, 23 Jan 2017 23:16:02 +0800 Subject: [PATCH 04/21] Dockerfile for stackage --- dockerfiles/stackage/Dockerfile | 20 ++++++++++++++++++++ stackage.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 dockerfiles/stackage/Dockerfile mode change 100644 => 100755 stackage.py diff --git a/dockerfiles/stackage/Dockerfile b/dockerfiles/stackage/Dockerfile new file mode 100644 index 0000000..22ed2bf --- /dev/null +++ b/dockerfiles/stackage/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.6 +MAINTAINER Justin Wong + +RUN echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie main contrib non-free" > /etc/apt/sources.list && \ + echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie-backports main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie-updates main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security/ jessie/updates main contrib non-free" >> /etc/apt/sources.list + +RUN apt-get update && \ + apt-get install -y git aria2 + +RUN pip3 install requests pyyaml + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && apt-get install -y locales -qq && locale-gen +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 + +ENV HOME=/tmp +CMD /bin/bash diff --git a/stackage.py b/stackage.py old mode 100644 new mode 100755 index 76d0f73..32193bf --- a/stackage.py +++ b/stackage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3 -u # python deps: requests, pyyaml # non-python deps: aria2, git From d96f7e5cf00bb45206cb529969ed80895d08bcaa Mon Sep 17 00:00:00 2001 From: bigeagle Date: Mon, 23 Jan 2017 23:35:05 +0800 Subject: [PATCH 05/21] stackage: improve code format and flush msg --- stackage.py | 61 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/stackage.py b/stackage.py index 32193bf..fe483f3 100755 --- a/stackage.py +++ b/stackage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 -u +#!/usr/bin/env python3 # python deps: requests, pyyaml # non-python deps: aria2, git @@ -20,7 +20,7 @@ class StackageSession(object): if force and file_path.is_file(): file_path.unlink() if file_path.is_file(): - print('{} exists, skipping'.format(file_path)) + print('{} exists, skipping'.format(file_path), flush=True) else: args = ['aria2c', url, '--dir={}'.format(dir_path)] if sha1: @@ -29,27 +29,43 @@ class StackageSession(object): file_path.unlink() raise OSError('Download failed for {}'.format(url)) else: - print('Downloaded {} to {}'.format(url, file_path)) + print('Downloaded {} to {}'.format(url, file_path), flush=True) def load_stack_setup(self): - d = yaml.load(requests.get( - 'https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml').content) + d = yaml.load( + requests + .get('https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml') + .content + ) for platform in d['ghc']: for ver in d['ghc'][platform]: - self.download('ghc', d['ghc'][platform][ver]['url'], d['ghc'][platform][ver]['sha1']) - d['ghc'][platform][ver]['url'] = 'http://mirrors.tuna.tsinghua.edu.cn/stackage/ghc/{}'.format( - d['ghc'][platform][ver]['url'].split('/')[-1]) - d['msys2'] = {'windows32': {'version': '20161025', - 'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/i686/msys2-base-i686-20161025.tar.xz', - 'content-length': 47526500, - 'sha1': '5D17FA53077A93A38A9AC0ACB8A03BF6C2FC32AD'}, - 'windows64': {'version': '20161025', - 'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz', - 'content-length': 47166584, - 'sha1': '05FD74A6C61923837DFFE22601C9014F422B5460'}} + self.download( + 'ghc', + d['ghc'][platform][ver]['url'], + d['ghc'][platform][ver]['sha1'], + ) + d['ghc'][platform][ver]['url'] = ( + 'http://mirrors.tuna.tsinghua.edu.cn/stackage/ghc/{}' + .format(d['ghc'][platform][ver]['url'].split('/')[-1]) + ) + + d['msys2'] = { + 'windows32': { + 'version': '20161025', + 'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/i686/msys2-base-i686-20161025.tar.xz', + 'content-length': 47526500, + 'sha1': '5D17FA53077A93A38A9AC0ACB8A03BF6C2FC32AD', + }, + 'windows64': { + 'version': '20161025', + 'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz', + 'content-length': 47166584, + 'sha1': '05FD74A6C61923837DFFE22601C9014F422B5460', + } + } with open(self._base_path / 'stack-setup.yaml', 'w') as f: yaml.dump(d, f) - print('Loaded stack-setup.yaml') + print('Loaded stack-setup.yaml', flush=True) def load_stackage_snapshots(self): for channel in ['lts-haskell', 'stackage-nightly']: @@ -59,9 +75,14 @@ class StackageSession(object): args = ['git', '-C', self._base_path, 'clone', '--depth', '1', 'https://github.com/fpco/{}.git'.format(channel)] subprocess.run(args, check=True) - print('Loaded {}'.format(channel)) - self.download('', 'https://www.stackage.org/download/snapshots.json', force=True) - print('Loaded snapshots.json') + print('Loaded {}'.format(channel), flush=True) + + self.download( + '', + 'https://www.stackage.org/download/snapshots.json', + force=True, + ) + print('Loaded snapshots.json', flush=True) if __name__ == '__main__': From 8f0df03e300a58a0dcd8dfd4ebf6fc6cd788c069 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Mon, 23 Jan 2017 23:44:20 +0800 Subject: [PATCH 06/21] make aria2 quiet --- stackage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stackage.py b/stackage.py index fe483f3..985227d 100755 --- a/stackage.py +++ b/stackage.py @@ -22,7 +22,10 @@ class StackageSession(object): if file_path.is_file(): print('{} exists, skipping'.format(file_path), flush=True) else: - args = ['aria2c', url, '--dir={}'.format(dir_path)] + args = [ + 'aria2c', url, '--dir={}'.format(dir_path), + '--file-allocation=none', '--quiet=true', + ] if sha1: args.append('--checksum=sha-1={}'.format(sha1)) if subprocess.run(args).returncode != 0 and file_path.is_file(): From dfe2e4f217b5e41be0d1eff4345887c5c469af16 Mon Sep 17 00:00:00 2001 From: Shao Cheng Date: Tue, 24 Jan 2017 00:07:51 +0800 Subject: [PATCH 07/21] Download to tmp file then move to dst. --- stackage.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/stackage.py b/stackage.py index 985227d..49a3684 100755 --- a/stackage.py +++ b/stackage.py @@ -6,6 +6,7 @@ import os import pathlib import requests +import shutil import subprocess import yaml @@ -23,22 +24,20 @@ class StackageSession(object): print('{} exists, skipping'.format(file_path), flush=True) else: args = [ - 'aria2c', url, '--dir={}'.format(dir_path), - '--file-allocation=none', '--quiet=true', + 'aria2c', url, '--dir={}'.format(dir_path), '--out={}.tmp'.format(url.split('/')[-1]), + '--file-allocation=none', '--quiet=true' ] if sha1: args.append('--checksum=sha-1={}'.format(sha1)) - if subprocess.run(args).returncode != 0 and file_path.is_file(): - file_path.unlink() - raise OSError('Download failed for {}'.format(url)) - else: - print('Downloaded {} to {}'.format(url, file_path), flush=True) + subprocess.run(args, check=True) + shutil.move('{}.tmp'.format(file_path), file_path) + print('Downloaded {} to {}'.format(url, file_path), flush=True) def load_stack_setup(self): d = yaml.load( requests - .get('https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml') - .content + .get('https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml') + .content ) for platform in d['ghc']: for ver in d['ghc'][platform]: @@ -49,7 +48,7 @@ class StackageSession(object): ) d['ghc'][platform][ver]['url'] = ( 'http://mirrors.tuna.tsinghua.edu.cn/stackage/ghc/{}' - .format(d['ghc'][platform][ver]['url'].split('/')[-1]) + .format(d['ghc'][platform][ver]['url'].split('/')[-1]) ) d['msys2'] = { From 42560cb28d03676dda15627ac02dc2c8e2bc5333 Mon Sep 17 00:00:00 2001 From: Wang Yuzhi Date: Tue, 24 Jan 2017 11:06:04 +0800 Subject: [PATCH 08/21] anaconda: fix msys2/noarch --- anaconda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anaconda.sh b/anaconda.sh index dc8cac1..2e82931 100755 --- a/anaconda.sh +++ b/anaconda.sh @@ -20,7 +20,7 @@ TMP_DIR=$(mktemp -d) CONDA_REPOS=("free" "r" "mro" "pro") CONDA_ARCHES=("noarch" "linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32") -CONDA_CLOUD_REPOS=("conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64" "conda-forge/noarch" "msys2/win-64") +CONDA_CLOUD_REPOS=("conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64" "conda-forge/noarch" "msys2/win-64" "msys2/noarch") EXIT_STATUS=0 EXIT_MSG="" From fb9050767140ed5b54eee6120bec16fce65ce34f Mon Sep 17 00:00:00 2001 From: Canto Ostinato Date: Tue, 14 Feb 2017 23:06:37 +0800 Subject: [PATCH 09/21] Fix bug in stackage.py Workaround a bug of stack which doesn't respect case insensitivity of checksums; remove unused clauses in setup info --- stackage.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stackage.py b/stackage.py index 49a3684..4e7c532 100755 --- a/stackage.py +++ b/stackage.py @@ -56,15 +56,17 @@ class StackageSession(object): 'version': '20161025', 'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/i686/msys2-base-i686-20161025.tar.xz', 'content-length': 47526500, - 'sha1': '5D17FA53077A93A38A9AC0ACB8A03BF6C2FC32AD', + 'sha1': '5d17fa53077a93a38a9ac0acb8a03bf6c2fc32ad', }, 'windows64': { 'version': '20161025', 'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz', 'content-length': 47166584, - 'sha1': '05FD74A6C61923837DFFE22601C9014F422B5460', + 'sha1': '05fd74a6c61923837dffe22601c9014f422b5460', } } + for i in ['portable-git', 'stack', 'ghcjs']: + del d[i] with open(self._base_path / 'stack-setup.yaml', 'w') as f: yaml.dump(d, f) print('Loaded stack-setup.yaml', flush=True) From 6bf020916c7afd31ea6e07241e1c9654a265d40f Mon Sep 17 00:00:00 2001 From: Xavier Yao Date: Fri, 24 Mar 2017 22:24:49 +0800 Subject: [PATCH 10/21] capture files for windows --- helpers/tf-xml-filelist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/tf-xml-filelist.py b/helpers/tf-xml-filelist.py index 68557d0..0899818 100755 --- a/helpers/tf-xml-filelist.py +++ b/helpers/tf-xml-filelist.py @@ -11,7 +11,7 @@ def get_filelist(xmlstring): for cnt in r.findall('s3:Contents', ns): key = cnt.find('s3:Key', ns) fname = key.text - if fname.endswith('/') or not (fname.startswith('linux') or fname.startswith('mac')): + if fname.endswith('/') or not (fname.startswith('linux') or fname.startswith('mac') or fname.startswith('windows')): continue size = cnt.find('s3:Size', ns).text From 29b269e837dd9d7842c2267fb7220753b1945e0c Mon Sep 17 00:00:00 2001 From: Xavier Yao Date: Sat, 25 Mar 2017 08:27:13 +0800 Subject: [PATCH 11/21] Update tf-gen-index.py --- helpers/tf-gen-index.py | 1 + 1 file changed, 1 insertion(+) diff --git a/helpers/tf-gen-index.py b/helpers/tf-gen-index.py index 9c528cc..0630eaf 100755 --- a/helpers/tf-gen-index.py +++ b/helpers/tf-gen-index.py @@ -22,6 +22,7 @@ def generate_fileindex(filelist): pythons = { 'linux': set([]), 'mac': set([]), + 'windows': set([]) } pkglist = [] From abfb37726f146a8aa2d75b377b28328a2382c28f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=82=88?= Date: Thu, 6 Apr 2017 01:41:22 +0800 Subject: [PATCH 12/21] add adobe-fonts --- adobe-fonts.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 adobe-fonts.sh diff --git a/adobe-fonts.sh b/adobe-fonts.sh new file mode 100755 index 0000000..c560926 --- /dev/null +++ b/adobe-fonts.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -e + +function repo_init() { + UPSTREAM="$1" + WORKING_DIR="$2" + git clone --mirror "$UPSTREAM" "$WORKING_DIR" +} + +function update_font_git() { + repo_dir="$1" + cd "$repo_dir" + echo "==== SYNC $repo_dir START ====" + /usr/bin/timeout -s INT 3600 git remote -v update + git repack -a -b -d + echo "==== SYNC $repo_dir DONE ====" +} + +function checkout_font_branch() { + repo_dir="$1" + work_tree="$2" + branch="$3" + echo "Checkout branch $branch to $work_tree" + if [[ ! -d "$2" ]]; then + git clone "$repo_dir" --branch "$branch" --single-branch "$work_tree" + else + cd "$work_tree" + git pull + fi +} + +UPSTREAM_BASE=${TUNASYNC_UPSTREAM_URL:-"https://github.com/adobe-fonts"} +REPOS=("source-code-pro" "source-sans-pro" "source-serif-pro" "source-han-sans" "source-han-serif") + +for repo in ${REPOS[@]}; do + if [[ ! -d "$TUNASYNC_WORKING_DIR/${repo}.git" ]]; then + echo "Initializing ${repo}.git" + repo_init "${UPSTREAM_BASE}/${repo}.git" "$TUNASYNC_WORKING_DIR/${repo}.git" + fi + update_font_git "$TUNASYNC_WORKING_DIR/${repo}.git" + checkout_font_branch "$TUNASYNC_WORKING_DIR/${repo}.git" "$TUNASYNC_WORKING_DIR/${repo}" "release" +done From 3b13366b1cb997c7780c6d94b050bc070f575957 Mon Sep 17 00:00:00 2001 From: Miao Wang Date: Thu, 6 Apr 2017 02:21:15 +0800 Subject: [PATCH 13/21] fix for missing uid inside docker container --- adobe-fonts.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/adobe-fonts.sh b/adobe-fonts.sh index c560926..a905e58 100755 --- a/adobe-fonts.sh +++ b/adobe-fonts.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +git_option="git -c user.email=non-existence@tuna.tsinghua.edu.cn -c user.name=Noname" + function repo_init() { UPSTREAM="$1" WORKING_DIR="$2" @@ -22,10 +24,10 @@ function checkout_font_branch() { branch="$3" echo "Checkout branch $branch to $work_tree" if [[ ! -d "$2" ]]; then - git clone "$repo_dir" --branch "$branch" --single-branch "$work_tree" + $git_option clone "$repo_dir" --branch "$branch" --single-branch "$work_tree" else cd "$work_tree" - git pull + $git_option pull fi } From b4b2e175f67dd705f347ed02ee8ce3d0b8305465 Mon Sep 17 00:00:00 2001 From: Hui Yiqun Date: Fri, 21 Apr 2017 11:15:18 +0800 Subject: [PATCH 14/21] aosp.sh: take use of TUNASYNC_UPSTREAM_URL --- aosp.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aosp.sh b/aosp.sh index 3f77d71..75bcb17 100755 --- a/aosp.sh +++ b/aosp.sh @@ -3,11 +3,12 @@ set -e REPO=${REPO:-"/usr/local/bin/repo"} USE_BITMAP_INDEX=${USE_BITMAP_INDEX:-"0"} +UPSTREAM=${TUNASYNC_UPSTREAM_URL:-"https://android.googlesource.com/mirror/manifest"} function repo_init() { mkdir -p $TUNASYNC_WORKING_DIR cd $TUNASYNC_WORKING_DIR - $REPO init -u https://android.googlesource.com/mirror/manifest --mirror + $REPO init -u $UPSTREAM --mirror } function repo_sync() { From c3b50863cf1e606b14e8dd6fbc19d2769947acbc Mon Sep 17 00:00:00 2001 From: Hui Yiqun Date: Sun, 23 Apr 2017 19:24:12 +0800 Subject: [PATCH 15/21] nodesource: add 7.x --- nodesource.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodesource.sh b/nodesource.sh index f5b6a72..51376af 100755 --- a/nodesource.sh +++ b/nodesource.sh @@ -13,7 +13,7 @@ function sync_nodesource() { DEB_BASE_URL="https://deb.nodesource.com" RPM_BASE_URL="https://rpm.nodesource.com" -node_versions=("0.10" "0.12" "4.x" "6.x") +node_versions=("0.10" "0.12" "4.x" "6.x" "7.x") declare success=true for ver in ${node_versions[@]}; do From 7d86c02349ff598ad8c191985ea29e6a0fafc47e Mon Sep 17 00:00:00 2001 From: Harry Chen Date: Sun, 28 May 2017 17:44:00 +0800 Subject: [PATCH 16/21] Add config for llvm repos Signed-off-by: Harry Chen --- llvm.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 llvm.sh diff --git a/llvm.sh b/llvm.sh new file mode 100755 index 0000000..aca707d --- /dev/null +++ b/llvm.sh @@ -0,0 +1,25 @@ +#!/bin/bash +function repo_init() { + UPSTREAM=$1 + WORKING_DIR=$2 + git clone --mirror $UPSTREAM $WORKING_DIR +} + +function repo_update() { + repo_dir="$1" + cd $repo_dir + echo "==== SYNC $repo_dir START ====" + /usr/bin/timeout -s INT 3600 git remote -v update + git repack -a -b -d + echo "==== SYNC $repo_dir DONE ====" +} + +repos=("llvm" "clang" "libcxx" "lldb" "clang-tools-extra" "polly" "zorg" "compiler-rt" "libcxxabi" "lld" "lnt") + +for repo in ${repos[@]}; do + if [[ ! -d "$TUNASYNC_WORKING_DIR/${repo}.git" ]]; then + echo "Initializing ${repo}.git" + repo_init "http://llvm.org/git/${repo}" "$TUNASYNC_WORKING_DIR/${repo}.git" + fi + repo_update "$TUNASYNC_WORKING_DIR/${repo}.git" +done From 6ec4545584b1c02a8c56c5d465019fc7506e6a5c Mon Sep 17 00:00:00 2001 From: Hui Yiqun Date: Thu, 1 Jun 2017 11:07:42 +0800 Subject: [PATCH 17/21] update for nodejs 8.x --- nodesource.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodesource.sh b/nodesource.sh index 51376af..74f449f 100755 --- a/nodesource.sh +++ b/nodesource.sh @@ -13,7 +13,7 @@ function sync_nodesource() { DEB_BASE_URL="https://deb.nodesource.com" RPM_BASE_URL="https://rpm.nodesource.com" -node_versions=("0.10" "0.12" "4.x" "6.x" "7.x") +node_versions=("0.10" "0.12" "4.x" "6.x" "7.x" "8.x") declare success=true for ver in ${node_versions[@]}; do From 398af382f160d29355679e3789997f0ff6f868ca Mon Sep 17 00:00:00 2001 From: Hui Yiqun Date: Thu, 1 Jun 2017 11:35:45 +0800 Subject: [PATCH 18/21] add script for raspberrypi-archive --- raspberrypi-archive.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 raspberrypi-archive.sh diff --git a/raspberrypi-archive.sh b/raspberrypi-archive.sh new file mode 100755 index 0000000..0d5de8d --- /dev/null +++ b/raspberrypi-archive.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://archive.raspberrypi.org/debian/"} + +cd "$TUNASYNC_WORKING_DIR" +lftp "$BASE_URL" -e "mirror --verbose -P 5 --delete --only-newer; bye" From 324b6806572061516dd6f54c7a2ddee5ba358858 Mon Sep 17 00:00:00 2001 From: Hui Yiqun Date: Sun, 20 Aug 2017 12:00:19 +0800 Subject: [PATCH 19/21] script to sync bioconductor --- bioconductor.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 bioconductor.sh diff --git a/bioconductor.sh b/bioconductor.sh new file mode 100755 index 0000000..34793ba --- /dev/null +++ b/bioconductor.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# requires: wget, rsync +# + +set -e +set -o pipefail + +UPSTREAM=${TUNASYNC_UPSTREAM_URL:-"rsync://master.bioconductor.org"} +VERSIONS=("3.5") +RELEASE_VERSION="3.5" + +RSYNC_OPTS="-aHvh --no-o --no-g --stats --exclude .~tmp~/ --delete --delete-after --delay-updates --safe-links --timeout=120 --contimeout=120" + +USE_IPV6=${USE_IPV6:-"0"} +if [[ $USE_IPV6 == "1" ]]; then + RSYNC_OPTS="-6 ${RSYNC_OPTS}" +fi + +mkdir -p ${TUNASYNC_WORKING_DIR}/packages + +for version in ${VERSIONS[@]}; do + upstream=${UPSTREAM}:${version} + dest=${TUNASYNC_WORKING_DIR}/packages/${version} + + [ ! -d "$dest" ] && mkdir -p "$dest" + + rsync ${RSYNC_OPTS} "$upstream" "$dest" +done + +ln -sfT ${TUNASYNC_WORKING_DIR}/packages/${RELEASE_VERSION} ${TUNASYNC_WORKING_DIR}/packages/release From 91e5a3ef7db8846803f5e8da75476e8eb9c89faa Mon Sep 17 00:00:00 2001 From: Hui Yiqun Date: Sun, 20 Aug 2017 12:30:51 +0800 Subject: [PATCH 20/21] bioconductor: fix url of upstream --- bioconductor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioconductor.sh b/bioconductor.sh index 34793ba..b0fe316 100755 --- a/bioconductor.sh +++ b/bioconductor.sh @@ -19,7 +19,7 @@ fi mkdir -p ${TUNASYNC_WORKING_DIR}/packages for version in ${VERSIONS[@]}; do - upstream=${UPSTREAM}:${version} + upstream=${UPSTREAM}/${version} dest=${TUNASYNC_WORKING_DIR}/packages/${version} [ ! -d "$dest" ] && mkdir -p "$dest" From e1946e5fb13ccc174f8df66be64faeb12aa24cbb Mon Sep 17 00:00:00 2001 From: Hui Yiqun Date: Sun, 20 Aug 2017 13:24:26 +0800 Subject: [PATCH 21/21] symlink: absolute path => relative path --- bioconductor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioconductor.sh b/bioconductor.sh index b0fe316..a55be5a 100755 --- a/bioconductor.sh +++ b/bioconductor.sh @@ -27,4 +27,4 @@ for version in ${VERSIONS[@]}; do rsync ${RSYNC_OPTS} "$upstream" "$dest" done -ln -sfT ${TUNASYNC_WORKING_DIR}/packages/${RELEASE_VERSION} ${TUNASYNC_WORKING_DIR}/packages/release +ln -sfT ${RELEASE_VERSION} ${TUNASYNC_WORKING_DIR}/packages/release