diff --git a/Dockerfile b/Dockerfile index 1c684de..45b8352 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,10 +7,12 @@ RUN echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie main contrib no echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security/ jessie/updates main contrib non-free" >> /etc/apt/sources.list RUN apt-get update && \ - apt-get install -y wget curl rsync lftp git jq python-dev python-pip yum-utils createrepo python3-dev python3-pip + apt-get install -y wget curl rsync lftp git jq python-dev python-pip yum-utils createrepo python3-dev python3-pip aria2 RUN pip install --upgrade pip setuptools && \ pip install bandersnatch==1.11 + +RUN pip3 install requests pyyaml RUN mkdir -p /home/tunasync-scripts ADD https://storage.googleapis.com/git-repo-downloads/repo /usr/local/bin/aosp-repo diff --git a/adobe-fonts.sh b/adobe-fonts.sh new file mode 100755 index 0000000..a905e58 --- /dev/null +++ b/adobe-fonts.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -e + +git_option="git -c user.email=non-existence@tuna.tsinghua.edu.cn -c user.name=Noname" + +function repo_init() { + UPSTREAM="$1" + WORKING_DIR="$2" + git clone --mirror "$UPSTREAM" "$WORKING_DIR" +} + +function update_font_git() { + repo_dir="$1" + cd "$repo_dir" + echo "==== SYNC $repo_dir START ====" + /usr/bin/timeout -s INT 3600 git remote -v update + git repack -a -b -d + echo "==== SYNC $repo_dir DONE ====" +} + +function checkout_font_branch() { + repo_dir="$1" + work_tree="$2" + branch="$3" + echo "Checkout branch $branch to $work_tree" + if [[ ! -d "$2" ]]; then + $git_option clone "$repo_dir" --branch "$branch" --single-branch "$work_tree" + else + cd "$work_tree" + $git_option pull + fi +} + +UPSTREAM_BASE=${TUNASYNC_UPSTREAM_URL:-"https://github.com/adobe-fonts"} +REPOS=("source-code-pro" "source-sans-pro" "source-serif-pro" "source-han-sans" "source-han-serif") + +for repo in ${REPOS[@]}; do + if [[ ! -d "$TUNASYNC_WORKING_DIR/${repo}.git" ]]; then + echo "Initializing ${repo}.git" + repo_init "${UPSTREAM_BASE}/${repo}.git" "$TUNASYNC_WORKING_DIR/${repo}.git" + fi + update_font_git "$TUNASYNC_WORKING_DIR/${repo}.git" + checkout_font_branch "$TUNASYNC_WORKING_DIR/${repo}.git" "$TUNASYNC_WORKING_DIR/${repo}" "release" +done diff --git a/anaconda.sh b/anaconda.sh index d6429dc..2e82931 100755 --- a/anaconda.sh +++ b/anaconda.sh @@ -18,12 +18,9 @@ LOCAL_DIR_BASE="${TUNASYNC_WORKING_DIR}" TMP_DIR=$(mktemp -d) CONDA_REPOS=("free" "r" "mro" "pro") -CONDA_ARCHES=("linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32") +CONDA_ARCHES=("noarch" "linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32") -CONDA_CLOUD_REPOS=( - "conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64" - "msys2/win-64" "r/linux-64" "r/osx-64" "r/osx-64" -) +CONDA_CLOUD_REPOS=("conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64" "conda-forge/noarch" "msys2/win-64" "msys2/noarch") EXIT_STATUS=0 EXIT_MSG="" diff --git a/aosp.sh b/aosp.sh index 3f77d71..75bcb17 100755 --- a/aosp.sh +++ b/aosp.sh @@ -3,11 +3,12 @@ set -e REPO=${REPO:-"/usr/local/bin/repo"} USE_BITMAP_INDEX=${USE_BITMAP_INDEX:-"0"} +UPSTREAM=${TUNASYNC_UPSTREAM_URL:-"https://android.googlesource.com/mirror/manifest"} function repo_init() { mkdir -p $TUNASYNC_WORKING_DIR cd $TUNASYNC_WORKING_DIR - $REPO init -u https://android.googlesource.com/mirror/manifest --mirror + $REPO init -u $UPSTREAM --mirror } function repo_sync() { diff --git a/bioconductor.sh b/bioconductor.sh new file mode 100755 index 0000000..a55be5a --- /dev/null +++ b/bioconductor.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# requires: wget, rsync +# + +set -e +set -o pipefail + +UPSTREAM=${TUNASYNC_UPSTREAM_URL:-"rsync://master.bioconductor.org"} +VERSIONS=("3.5") +RELEASE_VERSION="3.5" + +RSYNC_OPTS="-aHvh --no-o --no-g --stats --exclude .~tmp~/ --delete --delete-after --delay-updates --safe-links --timeout=120 --contimeout=120" + +USE_IPV6=${USE_IPV6:-"0"} +if [[ $USE_IPV6 == "1" ]]; then + RSYNC_OPTS="-6 ${RSYNC_OPTS}" +fi + +mkdir -p ${TUNASYNC_WORKING_DIR}/packages + +for version in ${VERSIONS[@]}; do + upstream=${UPSTREAM}/${version} + dest=${TUNASYNC_WORKING_DIR}/packages/${version} + + [ ! -d "$dest" ] && mkdir -p "$dest" + + rsync ${RSYNC_OPTS} "$upstream" "$dest" +done + +ln -sfT ${RELEASE_VERSION} ${TUNASYNC_WORKING_DIR}/packages/release diff --git a/dockerfiles/stackage/Dockerfile b/dockerfiles/stackage/Dockerfile new file mode 100644 index 0000000..22ed2bf --- /dev/null +++ b/dockerfiles/stackage/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.6 +MAINTAINER Justin Wong + +RUN echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie main contrib non-free" > /etc/apt/sources.list && \ + echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie-backports main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie-updates main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security/ jessie/updates main contrib non-free" >> /etc/apt/sources.list + +RUN apt-get update && \ + apt-get install -y git aria2 + +RUN pip3 install requests pyyaml + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && apt-get install -y locales -qq && locale-gen +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 + +ENV HOME=/tmp +CMD /bin/bash diff --git a/helpers/tf-gen-index.py b/helpers/tf-gen-index.py index 9c528cc..0630eaf 100755 --- a/helpers/tf-gen-index.py +++ b/helpers/tf-gen-index.py @@ -22,6 +22,7 @@ def generate_fileindex(filelist): pythons = { 'linux': set([]), 'mac': set([]), + 'windows': set([]) } pkglist = [] diff --git a/helpers/tf-xml-filelist.py b/helpers/tf-xml-filelist.py index 68557d0..0899818 100755 --- a/helpers/tf-xml-filelist.py +++ b/helpers/tf-xml-filelist.py @@ -11,7 +11,7 @@ def get_filelist(xmlstring): for cnt in r.findall('s3:Contents', ns): key = cnt.find('s3:Key', ns) fname = key.text - if fname.endswith('/') or not (fname.startswith('linux') or fname.startswith('mac')): + if fname.endswith('/') or not (fname.startswith('linux') or fname.startswith('mac') or fname.startswith('windows')): continue size = cnt.find('s3:Size', ns).text diff --git a/llvm.sh b/llvm.sh new file mode 100755 index 0000000..aca707d --- /dev/null +++ b/llvm.sh @@ -0,0 +1,25 @@ +#!/bin/bash +function repo_init() { + UPSTREAM=$1 + WORKING_DIR=$2 + git clone --mirror $UPSTREAM $WORKING_DIR +} + +function repo_update() { + repo_dir="$1" + cd $repo_dir + echo "==== SYNC $repo_dir START ====" + /usr/bin/timeout -s INT 3600 git remote -v update + git repack -a -b -d + echo "==== SYNC $repo_dir DONE ====" +} + +repos=("llvm" "clang" "libcxx" "lldb" "clang-tools-extra" "polly" "zorg" "compiler-rt" "libcxxabi" "lld" "lnt") + +for repo in ${repos[@]}; do + if [[ ! -d "$TUNASYNC_WORKING_DIR/${repo}.git" ]]; then + echo "Initializing ${repo}.git" + repo_init "http://llvm.org/git/${repo}" "$TUNASYNC_WORKING_DIR/${repo}.git" + fi + repo_update "$TUNASYNC_WORKING_DIR/${repo}.git" +done diff --git a/nodesource.sh b/nodesource.sh index f5b6a72..74f449f 100755 --- a/nodesource.sh +++ b/nodesource.sh @@ -13,7 +13,7 @@ function sync_nodesource() { DEB_BASE_URL="https://deb.nodesource.com" RPM_BASE_URL="https://rpm.nodesource.com" -node_versions=("0.10" "0.12" "4.x" "6.x") +node_versions=("0.10" "0.12" "4.x" "6.x" "7.x" "8.x") declare success=true for ver in ${node_versions[@]}; do diff --git a/raspberrypi-archive.sh b/raspberrypi-archive.sh new file mode 100755 index 0000000..0d5de8d --- /dev/null +++ b/raspberrypi-archive.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://archive.raspberrypi.org/debian/"} + +cd "$TUNASYNC_WORKING_DIR" +lftp "$BASE_URL" -e "mirror --verbose -P 5 --delete --only-newer; bye" diff --git a/stackage.py b/stackage.py new file mode 100755 index 0000000..4e7c532 --- /dev/null +++ b/stackage.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +# python deps: requests, pyyaml +# non-python deps: aria2, git + +import os +import pathlib +import requests +import shutil +import subprocess +import yaml + + +class StackageSession(object): + def __init__(self): + self._base_path = pathlib.Path(os.environ['TUNASYNC_WORKING_DIR']) + + def download(self, dir_, url, sha1=None, force=False): + dir_path = self._base_path / dir_ + file_path = dir_path / url.split('/')[-1] + if force and file_path.is_file(): + file_path.unlink() + if file_path.is_file(): + print('{} exists, skipping'.format(file_path), flush=True) + else: + args = [ + 'aria2c', url, '--dir={}'.format(dir_path), '--out={}.tmp'.format(url.split('/')[-1]), + '--file-allocation=none', '--quiet=true' + ] + if sha1: + args.append('--checksum=sha-1={}'.format(sha1)) + subprocess.run(args, check=True) + shutil.move('{}.tmp'.format(file_path), file_path) + print('Downloaded {} to {}'.format(url, file_path), flush=True) + + def load_stack_setup(self): + d = yaml.load( + requests + .get('https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml') + .content + ) + for platform in d['ghc']: + for ver in d['ghc'][platform]: + self.download( + 'ghc', + d['ghc'][platform][ver]['url'], + d['ghc'][platform][ver]['sha1'], + ) + d['ghc'][platform][ver]['url'] = ( + 'http://mirrors.tuna.tsinghua.edu.cn/stackage/ghc/{}' + .format(d['ghc'][platform][ver]['url'].split('/')[-1]) + ) + + d['msys2'] = { + 'windows32': { + 'version': '20161025', + 'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/i686/msys2-base-i686-20161025.tar.xz', + 'content-length': 47526500, + 'sha1': '5d17fa53077a93a38a9ac0acb8a03bf6c2fc32ad', + }, + 'windows64': { + 'version': '20161025', + 'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz', + 'content-length': 47166584, + 'sha1': '05fd74a6c61923837dffe22601c9014f422b5460', + } + } + for i in ['portable-git', 'stack', 'ghcjs']: + del d[i] + with open(self._base_path / 'stack-setup.yaml', 'w') as f: + yaml.dump(d, f) + print('Loaded stack-setup.yaml', flush=True) + + def load_stackage_snapshots(self): + for channel in ['lts-haskell', 'stackage-nightly']: + if (self._base_path / channel).is_dir(): + args = ['git', '-C', self._base_path / channel, 'pull'] + else: + args = ['git', '-C', self._base_path, 'clone', '--depth', '1', + 'https://github.com/fpco/{}.git'.format(channel)] + subprocess.run(args, check=True) + print('Loaded {}'.format(channel), flush=True) + + self.download( + '', + 'https://www.stackage.org/download/snapshots.json', + force=True, + ) + print('Loaded snapshots.json', flush=True) + + +if __name__ == '__main__': + s = StackageSession() + s.load_stackage_snapshots() + s.load_stack_setup()