From 1859fca56e2b10571deb592bc5d6539395b69ef4 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Sun, 1 May 2016 19:25:11 +0800 Subject: [PATCH] imported from tunasync --- anaconda.sh | 96 +++++++++++++++++++ aosp.sh | 21 +++++ docker.sh | 59 ++++++++++++ excludes/gen_debian_exclude.py | 13 +++ excludes/gen_kali_exclude.py | 13 +++ excludes/gen_ubuntu_ports_exclude.py | 13 +++ gitlab-ce.sh | 65 +++++++++++++ gitlab-ci-multi-runner.sh | 69 ++++++++++++++ hackage.sh | 92 +++++++++++++++++++ helpers/apt-download | 132 +++++++++++++++++++++++++++ homebrew.sh | 17 ++++ linux.sh | 12 +++ lxc-images.sh | 16 ++++ mongodb.sh | 88 ++++++++++++++++++ nodesource.sh | 18 ++++ openwrt.sh | 13 +++ pypi.sh | 9 ++ repo-ck.sh | 17 ++++ termux.sh | 16 ++++ 19 files changed, 779 insertions(+) create mode 100755 anaconda.sh create mode 100755 aosp.sh create mode 100755 docker.sh create mode 100644 excludes/gen_debian_exclude.py create mode 100644 excludes/gen_kali_exclude.py create mode 100644 excludes/gen_ubuntu_ports_exclude.py create mode 100755 gitlab-ce.sh create mode 100755 gitlab-ci-multi-runner.sh create mode 100755 hackage.sh create mode 100644 helpers/apt-download create mode 100755 homebrew.sh create mode 100755 linux.sh create mode 100755 lxc-images.sh create mode 100755 mongodb.sh create mode 100755 nodesource.sh create mode 100755 openwrt.sh create mode 100755 pypi.sh create mode 100755 repo-ck.sh create mode 100755 termux.sh diff --git a/anaconda.sh b/anaconda.sh new file mode 100755 index 0000000..bc53fad --- /dev/null +++ b/anaconda.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# requires: wget, lftp, jq +# + +set -e +set -o pipefail + +CONDA_REPO_BASE=${CONDA_REPO_BASE:-"http://repo.continuum.io"} +LOCAL_DIR_BASE="${TUNASYNC_WORKING_DIR}/pkgs" +TMP_DIR=$(mktemp -d) + +CONDA_REPOS=("free" "r" "mro" "pro") +CONDA_ARCHES=("linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32") + +function check-and-download() { + remote_file=$1 + local_file=$2 + wget -q --spider ${remote_file} + if [ $? -eq 0 ]; then + echo "downloading ${remote_file}" + wget -q -N -O ${local_file} ${remote_file} + return + fi + return 1 +} + +function cleanup () { + echo "cleaning up" + [ -d ${TMP_DIR} ] && { + [ -f ${TMP_DIR}/repodata.json ] && rm ${TMP_DIR}/repodata.json + [ -f ${TMP_DIR}/repodata.json.bz2 ] && rm ${TMP_DIR}/repodata.json.bz2 + rmdir ${TMP_DIR} + } +} + +trap cleanup EXIT + +echo ${TMP_DIR} + +for repo in ${CONDA_REPOS[@]}; do + for arch in ${CONDA_ARCHES[@]}; do + PKG_REPO_BASE="${CONDA_REPO_BASE}/pkgs/$repo/$arch" + repodata_url="${PKG_REPO_BASE}/repodata.json" + bz2_repodata_url="${PKG_REPO_BASE}/repodata.json.bz2" + LOCAL_DIR="${LOCAL_DIR_BASE}/$repo/$arch" + [ ! -d ${LOCAL_DIR} ] && mkdir -p ${LOCAL_DIR} + tmp_repodata="${TMP_DIR}/repodata.json" + tmp_bz2_repodata="${TMP_DIR}/repodata.json.bz2" + + check-and-download ${repodata_url} ${tmp_repodata} + check-and-download ${bz2_repodata_url} ${tmp_bz2_repodata} + + jq_cmd='.packages | to_entries[] | [.key, .value.size, .value.md5] | map(tostring) | join(" ")' + bzip2 -c -d ${tmp_bz2_repodata} | jq -r "${jq_cmd}" | while read line; + do + read -a tokens <<< $line + pkgfile=${tokens[0]} + pkgsize=${tokens[1]} + pkgmd5=${tokens[2]} + + pkg_url="${PKG_REPO_BASE}/${pkgfile}" + dest_file="${LOCAL_DIR}/${pkgfile}" + + declare downloaded=false + if [ -f ${dest_file} ]; then + rsize=`stat -c "%s" ${dest_file}` + if [ ${rsize} -eq ${pkgsize} ]; then + downloaded=true + echo "Skipping ${pkgfile}, size ${pkgsize}" + fi + fi + while [ $downloaded != true ]; do + echo "downloading ${pkg_url}" + wget -q -O ${dest_file} ${pkg_url} && { + # two space for md5sum check format + echo "${pkgmd5} ${dest_file}" | md5sum -c - && downloaded=true + } + done + done + + mv -f "${TMP_DIR}/repodata.json" "${LOCAL_DIR}/repodata.json" + mv -f "${TMP_DIR}/repodata.json.bz2" "${LOCAL_DIR}/repodata.json.bz2" + done +done + +function sync_installer() { + repo_url="$1" + repo_dir="$2" + + [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" + cd $repo_dir + lftp "${repo_url}/" -e "mirror --verbose -P 5; bye" +} + +sync_installer "${CONDA_REPO_BASE}/archive/" "${TUNASYNC_WORKING_DIR}/archive/" +sync_installer "${CONDA_REPO_BASE}/miniconda/" "${TUNASYNC_WORKING_DIR}/miniconda/" diff --git a/aosp.sh b/aosp.sh new file mode 100755 index 0000000..71ac834 --- /dev/null +++ b/aosp.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +REPO=${REPO:-"/usr/local/bin/repo"} + +function repo_init() { + mkdir -p $TUNASYNC_WORKING_DIR + cd $TUNASYNC_WORKING_DIR + $REPO init -u https://android.googlesource.com/mirror/manifest --mirror +} + +function repo_sync() { + cd $TUNASYNC_WORKING_DIR + $REPO sync -f +} + +if [ ! -d "$TUNASYNC_WORKING_DIR/git-repo.git" ]; then + echo "Initializing AOSP mirror" + repo_init +fi + +repo_sync diff --git a/docker.sh b/docker.sh new file mode 100755 index 0000000..a800e1f --- /dev/null +++ b/docker.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# reqires: wget, yum-utils + +set -e +set -o pipefail + +_here=`dirname $(realpath $0)` +. ${_here}/helpers/apt-download +APT_VERSIONS=("debian-wheezy" "debian-jessie" "ubuntu-precise" "ubuntu-trusty" "ubuntu-xenial") + +BASE_PATH="${TUNASYNC_WORKING_DIR}" +APT_PATH="${BASE_PATH}/apt/repo" +YUM_PATH="${BASE_PATH}/yum/repo" + +mkdir -p ${APT_PATH} ${YUM_PATH} + +wget -q -N -O ${BASE_PATH}/yum/gpg https://yum.dockerproject.org/gpg +wget -q -N -O ${BASE_PATH}/apt/gpg https://apt.dockerproject.org/gpg + +# YUM mirror +cache_dir="/tmp/yum-docker-cache/" +cfg="/tmp/docker-yum.conf" +cat < ${cfg} +[main] +keepcache=0 + +[centos6] +name=Docker Repository +baseurl=https://yum.dockerproject.org/repo/main/centos/6 +enabled=1 +gpgcheck=0 +gpgkey=https://yum.dockerproject.org/gpg +sslverify=0 + +[centos7] +name=Docker Repository +baseurl=https://yum.dockerproject.org/repo/main/centos/7 +enabled=1 +gpgcheck=0 +gpgkey=https://yum.dockerproject.org/gpg +sslverify=0 +EOF + +[ ! -d ${YUM_PATH}/centos6 ] && mkdir -p ${YUM_PATH}/centos6 +[ ! -d ${YUM_PATH}/centos7 ] && mkdir -p ${YUM_PATH}/centos7 +reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir +createrepo --update -v -c $cache_dir -o ${YUM_PATH}/centos6 ${YUM_PATH}/centos7 +createrepo --update -v -c $cache_dir -o ${YUM_PATH}/centos6 ${YUM_PATH}/centos7 +rm $cfg + +# APT mirror +base_url="http://apt.dockerproject.org/repo" +for version in ${APT_VERSIONS[@]}; do + apt-download-binary ${base_url} "$version" "main" "amd64" "${APT_PATH}" || true + apt-download-binary ${base_url} "$version" "main" "i386" "${APT_PATH}" || true +done + +# sync_docker "http://apt.dockerproject.org/" "${TUNASYNC_WORKING_DIR}/apt" +# sync_docker "http://yum.dockerproject.org/" "${TUNASYNC_WORKING_DIR}/yum" diff --git a/excludes/gen_debian_exclude.py b/excludes/gen_debian_exclude.py new file mode 100644 index 0000000..f48c28f --- /dev/null +++ b/excludes/gen_debian_exclude.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +ARCH_EXCLUDE = ['armel', 'alpha', 'hurd-i386', 'ia64', 'kfreebsd-amd64', 'kfreebsd-i386', 'mips', 'powerpc', 'ppc64el', 's390', 's390x', 'sparc'] + +CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes'] + +with open("debian-exclude.txt", 'wb') as f: + f.write(".~tmp~/\n") + f.write(".*\n") + for arch in ARCH_EXCLUDE: + for content in CONTENT_EXCLUDE: + f.write(content.format(arch=arch)) + f.write('\n') diff --git a/excludes/gen_kali_exclude.py b/excludes/gen_kali_exclude.py new file mode 100644 index 0000000..de5a031 --- /dev/null +++ b/excludes/gen_kali_exclude.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +ARCH_EXCLUDE = ['armel', 'armhf'] + +CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes'] + +with open("kali-exclude.txt", 'wb') as f: + f.write(".~tmp~/\n") + f.write(".*\n") + for arch in ARCH_EXCLUDE: + for content in CONTENT_EXCLUDE: + f.write(content.format(arch=arch)) + f.write('\n') diff --git a/excludes/gen_ubuntu_ports_exclude.py b/excludes/gen_ubuntu_ports_exclude.py new file mode 100644 index 0000000..9990ecf --- /dev/null +++ b/excludes/gen_ubuntu_ports_exclude.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +ARCH_EXCLUDE = ['powerpc', 'ppc64el', 'ia64', 'sparc', 'armel'] + +CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes'] + +with open("ubuntu-ports-exclude.txt", 'wb') as f: + f.write(".~tmp~/\n") + f.write(".*\n") + for arch in ARCH_EXCLUDE: + for content in CONTENT_EXCLUDE: + f.write(content.format(arch=arch)) + f.write('\n') diff --git a/gitlab-ce.sh b/gitlab-ce.sh new file mode 100755 index 0000000..0b5c42d --- /dev/null +++ b/gitlab-ce.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -e + +_here=`dirname $(realpath $0)` +. ${_here}/helpers/apt-download + +[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1) + +BASE_PATH="${TUNASYNC_WORKING_DIR}" + +YUM_PATH="${BASE_PATH}/yum" + +UBUNTU_VERSIONS=("trusty" "wily") +DEBIAN_VERSIONS=("wheezy" "jessie" "stretch") +UBUNTU_PATH="${BASE_PATH}/ubuntu/" +DEBIAN_PATH="${BASE_PATH}/debian/" + +mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH + +cache_dir="/tmp/yum-gitlab-ce-cache/" +cfg="/tmp/gitlab-ce-yum.conf" +cat < ${cfg} +[main] +keepcache=0 + +[el6] +name=el6 +baseurl=https://packages.gitlab.com/gitlab/gitlab-ce/el/6/x86_64 +repo_gpgcheck=0 +gpgcheck=0 +enabled=1 +gpgkey=https://packages.gitlab.com/gpg.key +sslverify=0 + +[el7] +name=el7 +baseurl=https://packages.gitlab.com/gitlab/gitlab-ce/el/7/x86_64 +repo_gpgcheck=0 +gpgcheck=0 +enabled=1 +gpgkey=https://packages.gitlab.com/gpg.key +sslverify=0 +EOF + +reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir +createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el6 ${YUM_PATH}/el6 +createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el7 ${YUM_PATH}/el7 +rm $cfg + +base_url="https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu" +for version in ${UBUNTU_VERSIONS[@]}; do + apt-download-binary ${base_url} "$version" "main" "amd64" "${UBUNTU_PATH}" || true + apt-download-binary ${base_url} "$version" "main" "i386" "${UBUNTU_PATH}" || true +done +echo "Ubuntu finished" + +base_url="https://packages.gitlab.com/gitlab/gitlab-ce/debian" +for version in ${DEBIAN_VERSIONS[@]}; do + apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" || true + apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" || true +done +echo "Debian finished" + + +# vim: ts=4 sts=4 sw=4 diff --git a/gitlab-ci-multi-runner.sh b/gitlab-ci-multi-runner.sh new file mode 100755 index 0000000..5c32569 --- /dev/null +++ b/gitlab-ci-multi-runner.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# reqires: wget, yum-utils +set -e +set -o pipefail + +_here=`dirname $(realpath $0)` +. ${_here}/helpers/apt-download + +[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1) + +BASE_PATH="${TUNASYNC_WORKING_DIR}" + +YUM_PATH="${BASE_PATH}/yum" + +UBUNTU_VERSIONS=("trusty" "xenial") +DEBIAN_VERSIONS=("wheezy" "jessie" "stretch") +UBUNTU_PATH="${BASE_PATH}/ubuntu/" +DEBIAN_PATH="${BASE_PATH}/debian/" + +mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH + +cache_dir="/tmp/yum-gitlab-runner-cache/" +cfg="/tmp/gitlab-runner-yum.conf" +cat < ${cfg} +[main] +keepcache=0 + +[el6] +name=gitlab-ci-multi-runner-el6 +baseurl=https://packages.gitlab.com/runner/gitlab-ci-multi-runner/el/6/x86_64 +repo_gpgcheck=0 +gpgcheck=0 +enabled=1 +gpgkey=https://packages.gitlab.com/gpg.key +sslverify=0 + +[el7] +name=gitlab-ci-multi-runner-el7 +baseurl=https://packages.gitlab.com/runner/gitlab-ci-multi-runner/el/7/x86_64 +repo_gpgcheck=0 +gpgcheck=0 +enabled=1 +gpgkey=https://packages.gitlab.com/gpg.key +sslverify=0 +EOF + +reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir +[ ! -d ${YUM_PATH}/el6 ] && mkdir -p ${YUM_PATH}/el6 +[ ! -d ${YUM_PATH}/el7 ] && mkdir -p ${YUM_PATH}/el7 +createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el6 ${YUM_PATH}/el6 +createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el7 ${YUM_PATH}/el7 +rm $cfg + +base_url="https://packages.gitlab.com/runner/gitlab-ci-multi-runner/ubuntu" +for version in ${UBUNTU_VERSIONS[@]}; do + apt-download-binary ${base_url} "$version" "main" "amd64" "${UBUNTU_PATH}" || true + apt-download-binary ${base_url} "$version" "main" "i386" "${UBUNTU_PATH}" || true +done +echo "Ubuntu finished" + +base_url="https://packages.gitlab.com/runner/gitlab-ci-multi-runner/debian" +for version in ${DEBIAN_VERSIONS[@]}; do + apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" || true + apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" || true +done +echo "Debian finished" + + +# vim: ts=4 sts=4 sw=4 diff --git a/hackage.sh b/hackage.sh new file mode 100755 index 0000000..2068274 --- /dev/null +++ b/hackage.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -e + +function remove_broken() { + interval=$1 + interval_file="/tmp/hackage_lastcheck" + now=`date +%s` + + if [[ -f ${interval_file} ]]; then + lastcheck=`cat ${interval_file}` + between=$(echo "${now}-${lastcheck}" | bc) + [[ $between -lt $interval ]] && echo "skip checking"; return 0 + fi + echo "start checking" + + mkdir -p "${TUNASYNC_WORKING_DIR}/package" + cd "${TUNASYNC_WORKING_DIR}/package" + + ls | while read line; do + echo -n "$line\t\t" + tar -tzf $line >/dev/null || (echo "FAIL"; rm $line) && echo "OK" + done + + echo `date +%s` > $interval_file +} + +function must_download() { + src=$1 + dst=$2 + while true; do + echo "downloading: $name" + wget "$src" -O "$dst" &>/dev/null || true + tar -tzf package/$name >/dev/null || rm package/$name && break + done +} + +function hackage_mirror() { + local_pklist="/tmp/hackage_local_pklist_$$.list" + remote_pklist="/tmp/hackage_remote_pklist_$$.list" + + cd ${TUNASYNC_WORKING_DIR} + mkdir -p package + + echo "Downloading index..." + rm index.tar.gz || true + axel http://hdiff.luite.com/packages/archive/index.tar.gz -o index.tar.gz > /dev/null + + echo "building local package list" + ls package | sed "s/\.tar\.gz$//" > $local_pklist + echo "preferred-versions" >> $local_pklist # ignore preferred-versions + + echo "building remote package list" + tar ztf index.tar.gz | (cut -d/ -f 1,2 2>/dev/null) | sed 's|/|-|' > $remote_pklist + + echo "building download list" + # substract local list from remote list + comm <(sort $remote_pklist) <(sort $local_pklist) -23 | while read pk; do + # limit concurrent level + bgcount=`jobs | wc -l` + while [[ $bgcount -ge 5 ]]; do + sleep 0.5 + bgcount=`jobs | wc -l` + done + + name="$pk.tar.gz" + if [ ! -a package/$name ]; then + must_download "http://hackage.haskell.org/package/$pk/$name" "package/$name" & + else + echo "skip existed: $name" + fi + done + + # delete redundanty files + comm <(sort $remote_pklist) <(sort $local_pklist) -13 | while read pk; do + name="$pk.tar.gz" + echo "deleting ${name}" + rm "package/$name" + done + + cp index.tar.gz 00-index.tar.gz +} + +function cleanup () { + echo "cleaning up" + [[ ! -z $local_pklist ]] && (rm $local_pklist $remote_pklist ; true) +} + +trap cleanup EXIT +remove_broken 86400 +hackage_mirror + +# vim: ts=4 sts=4 sw=4 diff --git a/helpers/apt-download b/helpers/apt-download new file mode 100644 index 0000000..f9d0f12 --- /dev/null +++ b/helpers/apt-download @@ -0,0 +1,132 @@ +#!/bin/bash +set -e +LOADED_APT_DOWNLOAD="yes" + +function check-and-download() { + remote_file=$1 + local_file=$2 + wget -q --spider ${remote_file} + if [ $? -eq 0 ]; then + echo "downloading ${remote_file}" + wget -q -N -O ${local_file} ${remote_file} + return + fi + return 0 +} + + +function apt-download-binary() { + base_url=$1 + dist=$2 + repo=$3 + arch=$4 + dest_base_dir=$5 + if [ -z $dest_base_dir ]; then + echo "Destination directory is empty, cannot continue" + return 1 + fi + + dest_dir="${dest_base_dir}/dists/${dist}" + [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" + check-and-download "${base_url}/dists/${dist}/Contents-${arch}.gz" "${dest_dir}/Contents-${arch}.gz" || true + check-and-download "${base_url}/dists/${dist}/InRelease" "${dest_dir}/InRelease" || true + check-and-download "${base_url}/dists/${dist}/Release" "${dest_dir}/Release" + check-and-download "${base_url}/dists/${dist}/Release.gpg" "${dest_dir}/Release.gpg" || true + + # Load Package Index URLs from Release file + release_file="${dest_dir}/Release" + dest_dir="${dest_base_dir}/dists/${dist}/${repo}/binary-${arch}" + [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" + + declare pkgidx_content="" + declare cnt_start=false + declare -i checksum_len + if (grep -e '^SHA256:$' ${release_file} &>/dev/null); then + checksum_cmd="sha256sum"; checksum_regex="^SHA256:$"; checksum_len=64 + elif (grep -e '^SHA1:$' ${release_file} &>/dev/null); then + checksum_cmd="sha1sum"; checksum_regex="^SHA1:$"; checksum_len=40 + elif (grep -e '^MD5Sum:$' ${release_file} &>/dev/null); then + checksum_cmd="md5sum"; checksum_regex="^MD5sum:$"; checksum_len=32 + fi + + while read line; do + if [[ ${cnt_start} = true ]]; then + read -a tokens <<< $line + checksum=${tokens[0]} + if [[ ${#checksum} != ${checksum_len} ]]; then + break + fi + filesize=${tokens[1]} + filename=${tokens[2]} + if [[ "$filename" =~ ${repo}/binary-${arch} ]]; then + # Load package list from Packages file + pkgidx_file="${dest_base_dir}/dists/${dist}/${filename}" + dest_dir=`dirname ${pkgidx_file}` + [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" + pkglist_url="${base_url}/dists/${dist}/${filename}" + check-and-download "${pkglist_url}" ${pkgidx_file} || true + echo "${checksum} ${pkgidx_file}" | ${checksum_cmd} -c - + if [ -z "${pkgidx_content}" -a -f ${pkgidx_file} ]; then + echo "getting packages index content" + case $filename in + "*.bz2") + pkgidx_content=`bunzip2 -c ${pkgidx_file}` + ;; + "*.gz") + pkgidx_content=`gunzip -c ${pkgidx_file}` + ;; + *) + pkgidx_content=`cat ${pkgidx_file}` + ;; + esac + fi + fi + else + if [[ "$line" =~ ${checksum_regex} ]]; then + cnt_start=true + fi + fi + done < ${release_file} + + if [ -z "${pkgidx_content}" ]; then + echo "index is empty, failed" + return 1 + fi + + # Set checksum method + if (echo -e "${pkgidx_content}" | grep -e '^SHA256' &>/dev/null); then + checksum_cmd="sha256sum"; checksum_regex="^SHA256" + elif (echo -e "${pkgidx_content}" | grep -e '^SHA1' &>/dev/null); then + checksum_cmd="sha1sum"; checksum_regex="^SHA1" + elif (echo -e "${pkgidx_content}" | grep -e '^MD5sum' &>/dev/null); then + checksum_cmd="md5sum"; checksum_regex="^MD5sum" + fi + + # Download packages + (echo -e "${pkgidx_content}" | grep -e '^Filename' -e '^Size' -e ${checksum_regex} | cut -d' ' -f 2) | \ + while read pkg_filename; read pkg_size; read pkg_checksum; do + dest_filename="${dest_base_dir}/${pkg_filename}" + dest_dir=`dirname ${dest_filename}` + [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir" + pkg_url="${base_url}/${pkg_filename}" + declare downloaded=false + if [ -f ${dest_filename} ]; then + rsize=`stat -c "%s" ${dest_filename}` + if [ ${rsize} -eq ${pkg_size} ]; then + downloaded=true + echo "Skipping ${pkg_filename}, size ${pkg_size}" + fi + fi + while [ $downloaded != true ]; do + echo "downloading ${pkg_url}" + wget -q -O ${dest_filename} ${pkg_url} && { + echo "${pkg_checksum} ${dest_filename}" | ${checksum_cmd} -c - && downloaded=true # two space for md5sum/sha1sum/sha256sum check format + } + done + done + + echo "Mirroring ${base_url} ${dist}, ${repo}, ${arch} done!" + +} + +# vim: ts=4 sts=4 sw=4 diff --git a/homebrew.sh b/homebrew.sh new file mode 100755 index 0000000..9e2abfb --- /dev/null +++ b/homebrew.sh @@ -0,0 +1,17 @@ +#!/bin/bash +if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then + echo "Directory not exists, fail" + exit 1 +fi + +function update_homebrew_git() { + repo_dir="$1" + cd $repo_dir + echo "==== SYNC $repo_dir START ====" + /usr/bin/timeout -s INT 3600 git remote -v update + echo "==== SYNC $repo_dir DONE ====" +} + +update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew.git" +update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew-python.git" +update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew-science.git" diff --git a/linux.sh b/linux.sh new file mode 100755 index 0000000..dca5e3c --- /dev/null +++ b/linux.sh @@ -0,0 +1,12 @@ +#!/bin/bash +if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then + echo "Directory not exists, fail" + exit 1 +fi + +function update_linux_git() { + cd $TUNASYNC_WORKING_DIR + /usr/bin/timeout -s INT 3600 git remote -v update +} + +update_linux_git diff --git a/lxc-images.sh b/lxc-images.sh new file mode 100755 index 0000000..9440a8c --- /dev/null +++ b/lxc-images.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +function sync_lxc_images() { + repo_url="$1" + repo_dir="$2" + + [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" + cd $repo_dir + + # lftp "${repo_url}/" -e "mirror --verbose --log=${TUNASYNC_LOG_FILE} --exclude-glob='*/SRPMS/*' -P 5 --delete --only-newer; bye" + lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye" +} + + +sync_lxc_images "http://images.linuxcontainers.org/images" "${TUNASYNC_WORKING_DIR}/images" +sync_lxc_images "http://images.linuxcontainers.org/meta" "${TUNASYNC_WORKING_DIR}/meta" diff --git a/mongodb.sh b/mongodb.sh new file mode 100755 index 0000000..444402f --- /dev/null +++ b/mongodb.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +_here=`dirname $(realpath $0)` +. ${_here}/helpers/apt-download + +[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1) + +BASE_PATH="${TUNASYNC_WORKING_DIR}" + +YUM_PATH="${BASE_PATH}/yum" +APT_PATH="${BASE_PATH}/apt" + +UBUNTU_VERSIONS=("trusty" "precise") +DEBIAN_VERSIONS=("wheezy") +MONGO_VERSIONS=("3.2" "3.0") +STABLE_VERSION="3.2" + +UBUNTU_PATH="${APT_PATH}/ubuntu" +DEBIAN_PATH="${APT_PATH}/debian" + +mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH + +cache_dir="/tmp/yum-mongodb-cache/" +cfg="/tmp/mongodb-yum.conf" +cat < ${cfg} +[main] +keepcache=0 + +EOF + +for mgver in ${MONGO_VERSIONS[@]}; do +cat <> ${cfg} +[el6-${mgver}] +name=el6-${mgver} +baseurl=https://repo.mongodb.org/yum/redhat/6/mongodb-org/${mgver}/x86_64/ +repo_gpgcheck=0 +gpgcheck=0 +enabled=1 +sslverify=0 + +[el7-${mgver}] +name=el7-${mgver} +baseurl=https://repo.mongodb.org/yum/redhat/7/mongodb-org/${mgver}/x86_64/ +repo_gpgcheck=0 +gpgcheck=0 +enabled=1 +sslverify=0 +EOF +done + +reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir +for mgver in ${MONGO_VERSIONS[@]}; do + createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el6-$mgver/ ${YUM_PATH}/el6-$mgver/ + createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el7-$mgver/ ${YUM_PATH}/el7-$mgver/ +done + +[ -e ${YUM_PATH}/el6 ] || (cd ${YUM_PATH}; ln -s el6-${STABLE_VERSION} el6) +[ -e ${YUM_PATH}/el7 ] || (cd ${YUM_PATH}; ln -s el7-${STABLE_VERSION} el7) + +rm $cfg + +base_url="http://repo.mongodb.org/apt/ubuntu" +for ubver in ${UBUNTU_VERSIONS[@]}; do + for mgver in ${MONGO_VERSIONS[@]}; do + version="$ubver/mongodb-org/$mgver" + apt-download-binary ${base_url} "$version" "multiverse" "amd64" "${UBUNTU_PATH}" || true + apt-download-binary ${base_url} "$version" "multiverse" "i386" "${UBUNTU_PATH}" || true + done + mg_basepath="${UBUNTU_PATH}/dists/$ubver/mongodb-org" + [ -e ${mg_basepath}/stable ] || (cd ${mg_basepath}; ln -s ${STABLE_VERSION} stable) +done +echo "Ubuntu finished" + +base_url="http://repo.mongodb.org/apt/debian" +for dbver in ${DEBIAN_VERSIONS[@]}; do + for mgver in ${MONGO_VERSIONS[@]}; do + version="$dbver/mongodb-org/$mgver" + apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" || true + apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" || true + done + mg_basepath="${DEBIAN_PATH}/dists/$dbver/mongodb-org" + [ -e ${mg_basepath}/stable ] || (cd ${mg_basepath}; ln -s ${STABLE_VERSION} stable) +done +echo "Debian finished" + + +# vim: ts=4 sts=4 sw=4 diff --git a/nodesource.sh b/nodesource.sh new file mode 100755 index 0000000..8dda8e7 --- /dev/null +++ b/nodesource.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +function sync_nodesource() { + repo_url="$1" + repo_dir="$2" + + [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" + cd $repo_dir + # lftp "${repo_url}/" -e "mirror --verbose --exclude-glob='*/SRPMS/*' -P 5 --delete --only-newer; bye" + lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye" +} + +sync_nodesource "https://deb.nodesource.com/node" "${TUNASYNC_WORKING_DIR}/deb" +sync_nodesource "https://deb.nodesource.com/node_0.12" "${TUNASYNC_WORKING_DIR}/deb_0.12" +sync_nodesource "https://deb.nodesource.com/node_4.x" "${TUNASYNC_WORKING_DIR}/deb_4.x" +sync_nodesource "https://rpm.nodesource.com/pub" "${TUNASYNC_WORKING_DIR}/rpm" +sync_nodesource "https://rpm.nodesource.com/pub_0.12" "${TUNASYNC_WORKING_DIR}/rpm_0.12" +sync_nodesource "https://rpm.nodesource.com/pub_4.x" "${TUNASYNC_WORKING_DIR}/rpm_4.x" diff --git a/openwrt.sh b/openwrt.sh new file mode 100755 index 0000000..372d696 --- /dev/null +++ b/openwrt.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +function sync_openwrt() { + repo_url="$1" + repo_dir="$2" + + [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" + cd $repo_dir + lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye" +} + +sync_openwrt "http://downloads.openwrt.org/chaos_calmer/15.05/" "${TUNASYNC_WORKING_DIR}/chaos_calmer/15.05" +sync_openwrt "http://downloads.openwrt.org/snapshots/trunk/" "${TUNASYNC_WORKING_DIR}/snapshots/trunk" diff --git a/pypi.sh b/pypi.sh new file mode 100755 index 0000000..5882e61 --- /dev/null +++ b/pypi.sh @@ -0,0 +1,9 @@ +#!/bin/bash +if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then + echo "Directory not exists, fail" + exit 1 +fi + +echo "Syncing to $TUNASYNC_WORKING_DIR" + +/usr/bin/timeout -s INT 3600 /home/tuna/.virtualenvs/bandersnatch/bin/bandersnatch -c /etc/bandersnatch.conf mirror || exit 1 diff --git a/repo-ck.sh b/repo-ck.sh new file mode 100755 index 0000000..8e4538a --- /dev/null +++ b/repo-ck.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +function sync_repo_ck() { + repo_url="$1" + repo_dir="$2" + + [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" + cd $repo_dir + lftp "${repo_url}/" -e 'mirror -v -P 5 --delete --only-missing --only-newer --no-recursion; bye' + wget "${repo_url}/repo-ck.db" -O "repo-ck.db" + wget "${repo_url}/repo-ck.files" -O "repo-ck.files" +} + +UPSTREAM="http://repo-ck.com" + +sync_repo_ck "${UPSTREAM}/x86_64" "${TUNASYNC_WORKING_DIR}/x86_64" +sync_repo_ck "${UPSTREAM}/i686" "${TUNASYNC_WORKING_DIR}/i686" diff --git a/termux.sh b/termux.sh new file mode 100755 index 0000000..9dd1002 --- /dev/null +++ b/termux.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +_here=`dirname $(realpath $0)` +. ${_here}/helpers/apt-download +[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1) + +BASE_PATH="${TUNASYNC_WORKING_DIR}" + +base_url="http://apt.termux.com" +ARCHES=("aarch64" "all" "arm" "i686") +for arch in ${ARCHES[@]}; do + echo "start syncing: ${arch}" + apt-download-binary "${base_url}" "stable" "main" "${arch}" "${BASE_PATH}" || true +done +echo "finished"