mirror of
https://github.com/tuna/tunasync-scripts.git
synced 2025-04-20 04:12:42 +00:00
Merge branch 'master' into dev
This commit is contained in:
bigeagle
commit
832510e9b9
@ -7,10 +7,12 @@ RUN echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie main contrib no
|
||||
echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security/ jessie/updates main contrib non-free" >> /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y wget curl rsync lftp git jq python-dev python-pip yum-utils createrepo python3-dev python3-pip
|
||||
apt-get install -y wget curl rsync lftp git jq python-dev python-pip yum-utils createrepo python3-dev python3-pip aria2
|
||||
|
||||
RUN pip install --upgrade pip setuptools && \
|
||||
pip install bandersnatch==1.11
|
||||
|
||||
RUN pip3 install requests pyyaml
|
||||
|
||||
RUN mkdir -p /home/tunasync-scripts
|
||||
ADD https://storage.googleapis.com/git-repo-downloads/repo /usr/local/bin/aosp-repo
|
||||
|
||||
44
adobe-fonts.sh
Executable file
44
adobe-fonts.sh
Executable file
@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
git_option="git -c user.email=non-existence@tuna.tsinghua.edu.cn -c user.name=Noname"
|
||||
|
||||
function repo_init() {
|
||||
UPSTREAM="$1"
|
||||
WORKING_DIR="$2"
|
||||
git clone --mirror "$UPSTREAM" "$WORKING_DIR"
|
||||
}
|
||||
|
||||
function update_font_git() {
|
||||
repo_dir="$1"
|
||||
cd "$repo_dir"
|
||||
echo "==== SYNC $repo_dir START ===="
|
||||
/usr/bin/timeout -s INT 3600 git remote -v update
|
||||
git repack -a -b -d
|
||||
echo "==== SYNC $repo_dir DONE ===="
|
||||
}
|
||||
|
||||
function checkout_font_branch() {
|
||||
repo_dir="$1"
|
||||
work_tree="$2"
|
||||
branch="$3"
|
||||
echo "Checkout branch $branch to $work_tree"
|
||||
if [[ ! -d "$2" ]]; then
|
||||
$git_option clone "$repo_dir" --branch "$branch" --single-branch "$work_tree"
|
||||
else
|
||||
cd "$work_tree"
|
||||
$git_option pull
|
||||
fi
|
||||
}
|
||||
|
||||
UPSTREAM_BASE=${TUNASYNC_UPSTREAM_URL:-"https://github.com/adobe-fonts"}
|
||||
REPOS=("source-code-pro" "source-sans-pro" "source-serif-pro" "source-han-sans" "source-han-serif")
|
||||
|
||||
for repo in ${REPOS[@]}; do
|
||||
if [[ ! -d "$TUNASYNC_WORKING_DIR/${repo}.git" ]]; then
|
||||
echo "Initializing ${repo}.git"
|
||||
repo_init "${UPSTREAM_BASE}/${repo}.git" "$TUNASYNC_WORKING_DIR/${repo}.git"
|
||||
fi
|
||||
update_font_git "$TUNASYNC_WORKING_DIR/${repo}.git"
|
||||
checkout_font_branch "$TUNASYNC_WORKING_DIR/${repo}.git" "$TUNASYNC_WORKING_DIR/${repo}" "release"
|
||||
done
|
||||
@ -18,12 +18,9 @@ LOCAL_DIR_BASE="${TUNASYNC_WORKING_DIR}"
|
||||
TMP_DIR=$(mktemp -d)
|
||||
|
||||
CONDA_REPOS=("free" "r" "mro" "pro")
|
||||
CONDA_ARCHES=("linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32")
|
||||
CONDA_ARCHES=("noarch" "linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32")
|
||||
|
||||
CONDA_CLOUD_REPOS=(
|
||||
"conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64"
|
||||
"msys2/win-64" "r/linux-64" "r/osx-64" "r/osx-64"
|
||||
)
|
||||
CONDA_CLOUD_REPOS=("conda-forge/linux-64" "conda-forge/osx-64" "conda-forge/win-64" "conda-forge/noarch" "msys2/win-64" "msys2/noarch")
|
||||
|
||||
EXIT_STATUS=0
|
||||
EXIT_MSG=""
|
||||
|
||||
3
aosp.sh
3
aosp.sh
@ -3,11 +3,12 @@ set -e
|
||||
|
||||
REPO=${REPO:-"/usr/local/bin/repo"}
|
||||
USE_BITMAP_INDEX=${USE_BITMAP_INDEX:-"0"}
|
||||
UPSTREAM=${TUNASYNC_UPSTREAM_URL:-"https://android.googlesource.com/mirror/manifest"}
|
||||
|
||||
function repo_init() {
|
||||
mkdir -p $TUNASYNC_WORKING_DIR
|
||||
cd $TUNASYNC_WORKING_DIR
|
||||
$REPO init -u https://android.googlesource.com/mirror/manifest --mirror
|
||||
$REPO init -u $UPSTREAM --mirror
|
||||
}
|
||||
|
||||
function repo_sync() {
|
||||
|
||||
30
bioconductor.sh
Executable file
30
bioconductor.sh
Executable file
@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
# requires: wget, rsync
|
||||
#
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
UPSTREAM=${TUNASYNC_UPSTREAM_URL:-"rsync://master.bioconductor.org"}
|
||||
VERSIONS=("3.5")
|
||||
RELEASE_VERSION="3.5"
|
||||
|
||||
RSYNC_OPTS="-aHvh --no-o --no-g --stats --exclude .~tmp~/ --delete --delete-after --delay-updates --safe-links --timeout=120 --contimeout=120"
|
||||
|
||||
USE_IPV6=${USE_IPV6:-"0"}
|
||||
if [[ $USE_IPV6 == "1" ]]; then
|
||||
RSYNC_OPTS="-6 ${RSYNC_OPTS}"
|
||||
fi
|
||||
|
||||
mkdir -p ${TUNASYNC_WORKING_DIR}/packages
|
||||
|
||||
for version in ${VERSIONS[@]}; do
|
||||
upstream=${UPSTREAM}/${version}
|
||||
dest=${TUNASYNC_WORKING_DIR}/packages/${version}
|
||||
|
||||
[ ! -d "$dest" ] && mkdir -p "$dest"
|
||||
|
||||
rsync ${RSYNC_OPTS} "$upstream" "$dest"
|
||||
done
|
||||
|
||||
ln -sfT ${RELEASE_VERSION} ${TUNASYNC_WORKING_DIR}/packages/release
|
||||
20
dockerfiles/stackage/Dockerfile
Normal file
20
dockerfiles/stackage/Dockerfile
Normal file
@ -0,0 +1,20 @@
|
||||
FROM python:3.6
|
||||
MAINTAINER Justin Wong <yuzhi.wang@tuna.tsinghua.edu.cn>
|
||||
|
||||
RUN echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie main contrib non-free" > /etc/apt/sources.list && \
|
||||
echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie-backports main contrib non-free" >> /etc/apt/sources.list && \
|
||||
echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian/ jessie-updates main contrib non-free" >> /etc/apt/sources.list && \
|
||||
echo "deb http://mirrors.tuna.tsinghua.edu.cn/debian-security/ jessie/updates main contrib non-free" >> /etc/apt/sources.list
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y git aria2
|
||||
|
||||
RUN pip3 install requests pyyaml
|
||||
|
||||
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && apt-get install -y locales -qq && locale-gen
|
||||
ENV LANG=en_US.UTF-8
|
||||
ENV LANGUAGE=en_US.UTF-8
|
||||
ENV LC_ALL=en_US.UTF-8
|
||||
|
||||
ENV HOME=/tmp
|
||||
CMD /bin/bash
|
||||
@ -22,6 +22,7 @@ def generate_fileindex(filelist):
|
||||
pythons = {
|
||||
'linux': set([]),
|
||||
'mac': set([]),
|
||||
'windows': set([])
|
||||
}
|
||||
pkglist = []
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ def get_filelist(xmlstring):
|
||||
for cnt in r.findall('s3:Contents', ns):
|
||||
key = cnt.find('s3:Key', ns)
|
||||
fname = key.text
|
||||
if fname.endswith('/') or not (fname.startswith('linux') or fname.startswith('mac')):
|
||||
if fname.endswith('/') or not (fname.startswith('linux') or fname.startswith('mac') or fname.startswith('windows')):
|
||||
continue
|
||||
|
||||
size = cnt.find('s3:Size', ns).text
|
||||
|
||||
25
llvm.sh
Executable file
25
llvm.sh
Executable file
@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
function repo_init() {
|
||||
UPSTREAM=$1
|
||||
WORKING_DIR=$2
|
||||
git clone --mirror $UPSTREAM $WORKING_DIR
|
||||
}
|
||||
|
||||
function repo_update() {
|
||||
repo_dir="$1"
|
||||
cd $repo_dir
|
||||
echo "==== SYNC $repo_dir START ===="
|
||||
/usr/bin/timeout -s INT 3600 git remote -v update
|
||||
git repack -a -b -d
|
||||
echo "==== SYNC $repo_dir DONE ===="
|
||||
}
|
||||
|
||||
repos=("llvm" "clang" "libcxx" "lldb" "clang-tools-extra" "polly" "zorg" "compiler-rt" "libcxxabi" "lld" "lnt")
|
||||
|
||||
for repo in ${repos[@]}; do
|
||||
if [[ ! -d "$TUNASYNC_WORKING_DIR/${repo}.git" ]]; then
|
||||
echo "Initializing ${repo}.git"
|
||||
repo_init "http://llvm.org/git/${repo}" "$TUNASYNC_WORKING_DIR/${repo}.git"
|
||||
fi
|
||||
repo_update "$TUNASYNC_WORKING_DIR/${repo}.git"
|
||||
done
|
||||
@ -13,7 +13,7 @@ function sync_nodesource() {
|
||||
DEB_BASE_URL="https://deb.nodesource.com"
|
||||
RPM_BASE_URL="https://rpm.nodesource.com"
|
||||
|
||||
node_versions=("0.10" "0.12" "4.x" "6.x")
|
||||
node_versions=("0.10" "0.12" "4.x" "6.x" "7.x" "8.x")
|
||||
declare success=true
|
||||
|
||||
for ver in ${node_versions[@]}; do
|
||||
|
||||
6
raspberrypi-archive.sh
Executable file
6
raspberrypi-archive.sh
Executable file
@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
BASE_URL=${TUNASYNC_UPSTREAM_URL:-"http://archive.raspberrypi.org/debian/"}
|
||||
|
||||
cd "$TUNASYNC_WORKING_DIR"
|
||||
lftp "$BASE_URL" -e "mirror --verbose -P 5 --delete --only-newer; bye"
|
||||
95
stackage.py
Executable file
95
stackage.py
Executable file
@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# python deps: requests, pyyaml
|
||||
# non-python deps: aria2, git
|
||||
|
||||
import os
|
||||
import pathlib
|
||||
import requests
|
||||
import shutil
|
||||
import subprocess
|
||||
import yaml
|
||||
|
||||
|
||||
class StackageSession(object):
|
||||
def __init__(self):
|
||||
self._base_path = pathlib.Path(os.environ['TUNASYNC_WORKING_DIR'])
|
||||
|
||||
def download(self, dir_, url, sha1=None, force=False):
|
||||
dir_path = self._base_path / dir_
|
||||
file_path = dir_path / url.split('/')[-1]
|
||||
if force and file_path.is_file():
|
||||
file_path.unlink()
|
||||
if file_path.is_file():
|
||||
print('{} exists, skipping'.format(file_path), flush=True)
|
||||
else:
|
||||
args = [
|
||||
'aria2c', url, '--dir={}'.format(dir_path), '--out={}.tmp'.format(url.split('/')[-1]),
|
||||
'--file-allocation=none', '--quiet=true'
|
||||
]
|
||||
if sha1:
|
||||
args.append('--checksum=sha-1={}'.format(sha1))
|
||||
subprocess.run(args, check=True)
|
||||
shutil.move('{}.tmp'.format(file_path), file_path)
|
||||
print('Downloaded {} to {}'.format(url, file_path), flush=True)
|
||||
|
||||
def load_stack_setup(self):
|
||||
d = yaml.load(
|
||||
requests
|
||||
.get('https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml')
|
||||
.content
|
||||
)
|
||||
for platform in d['ghc']:
|
||||
for ver in d['ghc'][platform]:
|
||||
self.download(
|
||||
'ghc',
|
||||
d['ghc'][platform][ver]['url'],
|
||||
d['ghc'][platform][ver]['sha1'],
|
||||
)
|
||||
d['ghc'][platform][ver]['url'] = (
|
||||
'http://mirrors.tuna.tsinghua.edu.cn/stackage/ghc/{}'
|
||||
.format(d['ghc'][platform][ver]['url'].split('/')[-1])
|
||||
)
|
||||
|
||||
d['msys2'] = {
|
||||
'windows32': {
|
||||
'version': '20161025',
|
||||
'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/i686/msys2-base-i686-20161025.tar.xz',
|
||||
'content-length': 47526500,
|
||||
'sha1': '5d17fa53077a93a38a9ac0acb8a03bf6c2fc32ad',
|
||||
},
|
||||
'windows64': {
|
||||
'version': '20161025',
|
||||
'url': 'http://mirrors.tuna.tsinghua.edu.cn/msys2/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz',
|
||||
'content-length': 47166584,
|
||||
'sha1': '05fd74a6c61923837dffe22601c9014f422b5460',
|
||||
}
|
||||
}
|
||||
for i in ['portable-git', 'stack', 'ghcjs']:
|
||||
del d[i]
|
||||
with open(self._base_path / 'stack-setup.yaml', 'w') as f:
|
||||
yaml.dump(d, f)
|
||||
print('Loaded stack-setup.yaml', flush=True)
|
||||
|
||||
def load_stackage_snapshots(self):
|
||||
for channel in ['lts-haskell', 'stackage-nightly']:
|
||||
if (self._base_path / channel).is_dir():
|
||||
args = ['git', '-C', self._base_path / channel, 'pull']
|
||||
else:
|
||||
args = ['git', '-C', self._base_path, 'clone', '--depth', '1',
|
||||
'https://github.com/fpco/{}.git'.format(channel)]
|
||||
subprocess.run(args, check=True)
|
||||
print('Loaded {}'.format(channel), flush=True)
|
||||
|
||||
self.download(
|
||||
'',
|
||||
'https://www.stackage.org/download/snapshots.json',
|
||||
force=True,
|
||||
)
|
||||
print('Loaded snapshots.json', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
s = StackageSession()
|
||||
s.load_stackage_snapshots()
|
||||
s.load_stack_setup()
|
||||
Loading…
x
Reference in New Issue
Block a user