yacloud/tunasync-scripts
1
0
Fork 0
mirror of https://github.com/tuna/tunasync-scripts.git synced 2025-04-20 04:12:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into dev

Browse Source
This commit is contained in:
bigeagle 2017-08-21 12:10:12 +08:00
commit 832510e9b9
12 changed files with 230 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Dockerfile
View File

@ -7,11 +7,13 @@ RUN echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie main contrib no
echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security/ jessie/updates main contrib non-free" >> /etc/apt/sources.list
RUN apt-get update && \
apt-get install -y wget curl rsync lftp git jq python-dev python-pip yum-utils createrepo python3-dev python3-pip
apt-get install -y wget curl rsync lftp git jq python-dev python-pip yum-utils createrepo python3-dev python3-pip aria2
RUN pip install --upgrade pip setuptools && \
pip install bandersnatch==1.11
RUN pip3 install requests pyyaml
RUN mkdir -p /home/tunasync-scripts
ADD https://storage.googleapis.com/git-repo-downloads/repo /usr/local/bin/aosp-repo
RUN chmod a+x /usr/local/bin/aosp-repo

44
adobe-fonts.sh Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
set -e
git_option="git -c user.email=non-existence@tuna.tsinghua.edu.cn -c user.name=Noname"
function repo_init() {
UPSTREAM="$1"
WORKING_DIR="$2"
git clone --mirror "$UPSTREAM" "$WORKING_DIR"
}
function update_font_git() {
repo_dir="$1"
cd "$repo_dir"
echo "==== SYNC $repo_dir START ===="
/usr/bin/timeout -s INT 3600 git remote -v update
git repack -a -b -d
echo "==== SYNC $repo_dir DONE ===="
}
function checkout_font_branch() {
repo_dir="$1"
work_tree="$2"
branch="$3"
echo "Checkout branch $branch to $work_tree"
if [[ ! -d "$2" ]]; then
$git_option clone "$repo_dir" --branch "$branch" --single-branch "$work_tree"
else
cd "$work_tree"
$git_option pull
fi
}
UPSTREAM_BASE=${TUNASYNC_UPSTREAM_URL:-"https://github.com/adobe-fonts"}
REPOS=("source-code-pro" "source-sans-pro" "source-serif-pro" "source-han-sans" "source-han-serif")
for repo in ${REPOS[@]}; do
if [[ ! -d "$TUNASYNC_WORKING_DIR/${repo}.git" ]]; then
echo "Initializing ${repo}.git"
repo_init "${UPSTREAM_BASE}/${repo}.git" "$TUNASYNC_WORKING_DIR/${repo}.git"
fi
update_font_git "$TUNASYNC_WORKING_DIR/${repo}.git"
checkout_font_branch "$TUNASYNC_WORKING_DIR/${repo}.git" "$TUNASYNC_WORKING_DIR/${repo}" "release"
done

7
anaconda.sh
View File

@ -18,12 +18,9 @@ LOCAL_DIR_BASE="${TUNASYNC_WORKING_DIR}"
TMP_DIR=$(mktemp -d)
CONDA_REPOS=("free" "r" "mro" "pro")
CONDA_ARCHES=("linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32")
CONDA_ARCHES=("noarch" "linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32")
CONDA_CLOUD_REPOS=(
"conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64"
"msys2/win-64" "r/linux-64" "r/osx-64" "r/osx-64"
)
CONDA_CLOUD_REPOS=("conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64" "conda-forge/noarch" "msys2/win-64" "msys2/noarch")
EXIT_STATUS=0
EXIT_MSG=""

3
aosp.sh
View File

@ -3,11 +3,12 @@ set -e
REPO=${REPO:-"/usr/local/bin/repo"}
USE_BITMAP_INDEX=${USE_BITMAP_INDEX:-"0"}
UPSTREAM=${TUNASYNC_UPSTREAM_URL:-"https://android.googlesource.com/mirror/manifest"}
function repo_init() {
mkdir -p $TUNASYNC_WORKING_DIR
cd $TUNASYNC_WORKING_DIR
$REPO init -u https://android.googlesource.com/mirror/manifest --mirror
$REPO init -u $UPSTREAM --mirror
}
function repo_sync() {

30
bioconductor.sh Executable file
View File

@ -0,0 +1,30 @@
#!/bin/bash
# requires: wget, rsync
#
set -e
set -o pipefail
UPSTREAM=${TUNASYNC_UPSTREAM_URL:-"rsync://master.bioconductor.org"}
VERSIONS=("3.5")
RELEASE_VERSION="3.5"
RSYNC_OPTS="-aHvh --no-o --no-g --stats --exclude .~tmp~/ --delete --delete-after --delay-updates --safe-links --timeout=120 --contimeout=120"
USE_IPV6=${USE_IPV6:-"0"}
if [[ $USE_IPV6 == "1" ]]; then
RSYNC_OPTS="-6 ${RSYNC_OPTS}"
fi
mkdir -p ${TUNASYNC_WORKING_DIR}/packages
for version in ${VERSIONS[@]}; do
upstream=${UPSTREAM}/${version}
dest=${TUNASYNC_WORKING_DIR}/packages/${version}
[ ! -d "$dest" ] && mkdir -p "$dest"
rsync ${RSYNC_OPTS} "$upstream" "$dest"
done
ln -sfT ${RELEASE_VERSION} ${TUNASYNC_WORKING_DIR}/packages/release

20
dockerfiles/stackage/Dockerfile Normal file
View File

@ -0,0 +1,20 @@
FROM python:3.6
MAINTAINER Justin Wong <yuzhi.wang@tuna.tsinghua.edu.cn>
RUN echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie main contrib non-free" > /etc/apt/sources.list && \
echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie-backports main contrib non-free" >> /etc/apt/sources.list && \
echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie-updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security/ jessie/updates main contrib non-free" >> /etc/apt/sources.list
RUN apt-get update && \
apt-get install -y git aria2
RUN pip3 install requests pyyaml
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && apt-get install -y locales -qq && locale-gen
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US.UTF-8
ENV LC_ALL=en_US.UTF-8
ENV HOME=/tmp
CMD /bin/bash

1
helpers/tf-gen-index.py
View File

@ -22,6 +22,7 @@ def generate_fileindex(filelist):
pythons = {
'linux': set([]),
'mac': set([]),
'windows': set([])
}
pkglist = []

2
helpers/tf-xml-filelist.py
View File

@ -11,7 +11,7 @@ def get_filelist(xmlstring):
for cnt in r.findall('s3:Contents', ns):
key = cnt.find('s3:Key', ns)
fname = key.text
if fname.endswith('/') or not (fname.startswith('linux') or fname.startswith('mac')):
if fname.endswith('/') or not (fname.startswith('linux') or fname.startswith('mac') or fname.startswith('windows')):
continue
size = cnt.find('s3:Size', ns).text

25
llvm.sh Executable file
View File

@ -0,0 +1,25 @@
#!/bin/bash
function repo_init() {
UPSTREAM=$1
WORKING_DIR=$2
git clone --mirror $UPSTREAM $WORKING_DIR
}
function repo_update() {
repo_dir="$1"
cd $repo_dir
echo "==== SYNC $repo_dir START ===="
/usr/bin/timeout -s INT 3600 git remote -v update
git repack -a -b -d
echo "==== SYNC $repo_dir DONE ===="
}
repos=("llvm" "clang" "libcxx" "lldb" "clang-tools-extra" "polly" "zorg" "compiler-rt" "libcxxabi" "lld" "lnt")
for repo in ${repos[@]}; do
if [[ ! -d "$TUNASYNC_WORKING_DIR/${repo}.git" ]]; then
echo "Initializing ${repo}.git"
repo_init "http://llvm.org/git/${repo}" "$TUNASYNC_WORKING_DIR/${repo}.git"
fi
repo_update "$TUNASYNC_WORKING_DIR/${repo}.git"
done

2
nodesource.sh
View File

@ -13,7 +13,7 @@ function sync_nodesource() {
DEB_BASE_URL="https://deb.nodesource.com"
RPM_BASE_URL="https://rpm.nodesource.com"
node_versions=("0.10" "0.12" "4.x" "6.x")
node_versions=("0.10" "0.12" "4.x" "6.x" "7.x" "8.x")
declare success=true
for ver in ${node_versions[@]}; do

6
raspberrypi-archive.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/bash
BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://archive.raspberrypi.org/debian/"}
cd "$TUNASYNC_WORKING_DIR"
lftp "$BASE_URL" -e "mirror --verbose -P 5 --delete --only-newer; bye"

95
stackage.py Executable file
View File

@ -0,0 +1,95 @@
#!/usr/bin/env python3
# python deps: requests, pyyaml
# non-python deps: aria2, git
import os
import pathlib
import requests
import shutil
import subprocess
import yaml
class StackageSession(object):
def __init__(self):
self._base_path = pathlib.Path(os.environ['TUNASYNC_WORKING_DIR'])
def download(self, dir_, url, sha1=None, force=False):
dir_path = self._base_path / dir_
file_path = dir_path / url.split('/')[-1]
if force and file_path.is_file():
file_path.unlink()
if file_path.is_file():
print('{} exists, skipping'.format(file_path), flush=True)
else:
args = [
'aria2c', url, '--dir={}'.format(dir_path), '--out={}.tmp'.format(url.split('/')[-1]),
'--file-allocation=none', '--quiet=true'
]
if sha1:
args.append('--checksum=sha-1={}'.format(sha1))
subprocess.run(args, check=True)
shutil.move('{}.tmp'.format(file_path), file_path)
print('Downloaded {} to {}'.format(url, file_path), flush=True)
def load_stack_setup(self):
d = yaml.load(
requests
.get('https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml')
.content
)
for platform in d['ghc']:
for ver in d['ghc'][platform]:
self.download(
'ghc',
d['ghc'][platform][ver]['url'],
d['ghc'][platform][ver]['sha1'],
)
d['ghc'][platform][ver]['url'] = (
'http://mirrors.tuna.tsinghua.edu.cn/stackage/ghc/{}'
.format(d['ghc'][platform][ver]['url'].split('/')[-1])
)
d['msys2'] = {
'windows32': {
'version': '20161025',
'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/i686/msys2-base-i686-20161025.tar.xz',
'content-length': 47526500,
'sha1': '5d17fa53077a93a38a9ac0acb8a03bf6c2fc32ad',
},
'windows64': {
'version': '20161025',
'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz',
'content-length': 47166584,
'sha1': '05fd74a6c61923837dffe22601c9014f422b5460',
}
}
for i in ['portable-git', 'stack', 'ghcjs']:
del d[i]
with open(self._base_path / 'stack-setup.yaml', 'w') as f:
yaml.dump(d, f)
print('Loaded stack-setup.yaml', flush=True)
def load_stackage_snapshots(self):
for channel in ['lts-haskell', 'stackage-nightly']:
if (self._base_path / channel).is_dir():
args = ['git', '-C', self._base_path / channel, 'pull']
else:
args = ['git', '-C', self._base_path, 'clone', '--depth', '1',
'https://github.com/fpco/{}.git'.format(channel)]
subprocess.run(args, check=True)
print('Loaded {}'.format(channel), flush=True)
self.download(
'',
'https://www.stackage.org/download/snapshots.json',
force=True,
)
print('Loaded snapshots.json', flush=True)
if __name__ == '__main__':
s = StackageSession()
s.load_stackage_snapshots()
s.load_stack_setup()