From 8ae38ff406bac491a44ca183be4cbce25f8d61e3 Mon Sep 17 00:00:00 2001 From: bigeagle Date: Tue, 19 May 2015 20:15:07 +0800 Subject: [PATCH] update scripts --- scripts/android.sh | 10 +++++++++- scripts/gen_debian_exclude.py | 4 +++- scripts/gen_kali_exclude.py | 13 +++++++++++++ scripts/gen_ubuntu_ports_exclude.py | 13 +++++++++++++ scripts/homebrew.sh | 9 +++++++-- scripts/lxc-images.sh | 13 +++++++++++++ scripts/nodesource.sh | 14 ++++++++------ scripts/pypi.sh | 4 +++- scripts/repo-ck.sh | 12 ++++++------ scripts/ubuntu_debmirror.sh | 27 +++++++++++++++++++++++++++ 10 files changed, 102 insertions(+), 17 deletions(-) create mode 100644 scripts/gen_kali_exclude.py create mode 100644 scripts/gen_ubuntu_ports_exclude.py create mode 100755 scripts/lxc-images.sh create mode 100644 scripts/ubuntu_debmirror.sh diff --git a/scripts/android.sh b/scripts/android.sh index bf296fe..efb2fd3 100755 --- a/scripts/android.sh +++ b/scripts/android.sh @@ -2,7 +2,15 @@ function sync_android() { cd $TUNASYNC_WORKING_DIR - /usr/local/bin/android-repo sync + /usr/local/bin/android-repo sync -f +} + +function update_server_info() { + for repo in $(find $TUNASYNC_WORKING_DIR -type d -not -path "*/.repo/*" -name "*.git") + do + cd $repo + git update-server-info + done } sync_android diff --git a/scripts/gen_debian_exclude.py b/scripts/gen_debian_exclude.py index e04ad24..f48c28f 100644 --- a/scripts/gen_debian_exclude.py +++ b/scripts/gen_debian_exclude.py @@ -1,10 +1,12 @@ #!/usr/bin/env python -ARCH_EXCLUDE = ['armel', 'arm64', 'alpha', 'hurd-i386', 'ia64', 'kfreebsd-amd64', 'kfreebsd-i386', 'mips', 'powerpc', 'ppc64el', 's390', 's390x', 'sparc'] +ARCH_EXCLUDE = ['armel', 'alpha', 'hurd-i386', 'ia64', 'kfreebsd-amd64', 'kfreebsd-i386', 'mips', 'powerpc', 'ppc64el', 's390', 's390x', 'sparc'] CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes'] with open("debian-exclude.txt", 'wb') as f: + f.write(".~tmp~/\n") + f.write(".*\n") for arch in ARCH_EXCLUDE: for content in CONTENT_EXCLUDE: f.write(content.format(arch=arch)) diff --git a/scripts/gen_kali_exclude.py b/scripts/gen_kali_exclude.py new file mode 100644 index 0000000..de5a031 --- /dev/null +++ b/scripts/gen_kali_exclude.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +ARCH_EXCLUDE = ['armel', 'armhf'] + +CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes'] + +with open("kali-exclude.txt", 'wb') as f: + f.write(".~tmp~/\n") + f.write(".*\n") + for arch in ARCH_EXCLUDE: + for content in CONTENT_EXCLUDE: + f.write(content.format(arch=arch)) + f.write('\n') diff --git a/scripts/gen_ubuntu_ports_exclude.py b/scripts/gen_ubuntu_ports_exclude.py new file mode 100644 index 0000000..9990ecf --- /dev/null +++ b/scripts/gen_ubuntu_ports_exclude.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +ARCH_EXCLUDE = ['powerpc', 'ppc64el', 'ia64', 'sparc', 'armel'] + +CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes'] + +with open("ubuntu-ports-exclude.txt", 'wb') as f: + f.write(".~tmp~/\n") + f.write(".*\n") + for arch in ARCH_EXCLUDE: + for content in CONTENT_EXCLUDE: + f.write(content.format(arch=arch)) + f.write('\n') diff --git a/scripts/homebrew.sh b/scripts/homebrew.sh index 352a67b..9e2abfb 100755 --- a/scripts/homebrew.sh +++ b/scripts/homebrew.sh @@ -5,8 +5,13 @@ if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then fi function update_homebrew_git() { - cd $TUNASYNC_WORKING_DIR + repo_dir="$1" + cd $repo_dir + echo "==== SYNC $repo_dir START ====" /usr/bin/timeout -s INT 3600 git remote -v update + echo "==== SYNC $repo_dir DONE ====" } -update_homebrew_git +update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew.git" +update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew-python.git" +update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew-science.git" diff --git a/scripts/lxc-images.sh b/scripts/lxc-images.sh new file mode 100755 index 0000000..09ce1ce --- /dev/null +++ b/scripts/lxc-images.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +function sync_lxc_images() { + repo_url="$1" + repo_dir="$2" + cd $repo_dir + + # lftp "${repo_url}/" -e "mirror --verbose --log=${TUNASYNC_LOG_FILE} --exclude-glob='*/SRPMS/*' -P 5 --delete --only-newer; bye" + lftp "${repo_url}/" -e "mirror --verbose --exclude lxd/ -P 5 --delete --only-newer; bye" +} + + +sync_lxc_images "http://images.linuxcontainers.org/" "${TUNASYNC_WORKING_DIR}/" diff --git a/scripts/nodesource.sh b/scripts/nodesource.sh index 296caa9..89c6c1f 100644 --- a/scripts/nodesource.sh +++ b/scripts/nodesource.sh @@ -1,11 +1,13 @@ #!/bin/bash -function sync_repo_ck() { - repo_url="$1" - repo_dir="$2" - [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" - cd $repo_dir - lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye" +function sync_repo_ck() { + repo_url="$1" + repo_dir="$2" + + [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" + cd $repo_dir + # lftp "${repo_url}/" -e "mirror --verbose --log=${TUNASYNC_LOG_FILE} --exclude-glob='*/SRPMS/*' -P 5 --delete --only-newer; bye" + lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye" } sync_repo_ck "https://deb.nodesource.com/node" "${TUNASYNC_WORKING_DIR}/deb" diff --git a/scripts/pypi.sh b/scripts/pypi.sh index 7989dc1..5882e61 100755 --- a/scripts/pypi.sh +++ b/scripts/pypi.sh @@ -4,4 +4,6 @@ if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then exit 1 fi -/usr/bin/timeout -s INT 3600 /home/tuna/.virtualenvs/bandersnatch/bin/bandersnatch -c /etc/bandersnatch.conf mirror +echo "Syncing to $TUNASYNC_WORKING_DIR" + +/usr/bin/timeout -s INT 3600 /home/tuna/.virtualenvs/bandersnatch/bin/bandersnatch -c /etc/bandersnatch.conf mirror || exit 1 diff --git a/scripts/repo-ck.sh b/scripts/repo-ck.sh index bae40b8..8e4538a 100755 --- a/scripts/repo-ck.sh +++ b/scripts/repo-ck.sh @@ -6,12 +6,12 @@ function sync_repo_ck() { [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" cd $repo_dir - lftp "${repo_url}/" -e 'mirror -v -P 5 --delete --only-missing --only-newer --no-recursion; bye' || return 1 - wget "${repo_url}/repo-ck.db" -O "repo-ck.db" || return 1 + lftp "${repo_url}/" -e 'mirror -v -P 5 --delete --only-missing --only-newer --no-recursion; bye' + wget "${repo_url}/repo-ck.db" -O "repo-ck.db" wget "${repo_url}/repo-ck.files" -O "repo-ck.files" } -stat=0 -sync_repo_ck "${TUNASYNC_UPSTREAM_URL}/x86_64" "${TUNASYNC_WORKING_DIR}/x86_64" || stat=1 -sync_repo_ck "${TUNASYNC_UPSTREAM_URL}/i686" "${TUNASYNC_WORKING_DIR}/i686" || stat=1 -exit $stat +UPSTREAM="http://repo-ck.com" + +sync_repo_ck "${UPSTREAM}/x86_64" "${TUNASYNC_WORKING_DIR}/x86_64" +sync_repo_ck "${UPSTREAM}/i686" "${TUNASYNC_WORKING_DIR}/i686" diff --git a/scripts/ubuntu_debmirror.sh b/scripts/ubuntu_debmirror.sh new file mode 100644 index 0000000..da5fc94 --- /dev/null +++ b/scripts/ubuntu_debmirror.sh @@ -0,0 +1,27 @@ +#!/bin/bash +SYNC_FILES="$TUNASYNC_WORKING_DIR" +# SYNC_FILES="/srv/mirror_disk/ubuntu/_working/" +#LOG_FILE="$TUNASYNC_LOG_FILE" + +# [ -f $SYNC_LOCK ] && exit 1 +# touch $SYNC_LOCK + + +echo ">> Starting sync on $(date --rfc-3339=seconds)" + +arch="i386,amd64" +sections="main,main/debian-installer,multiverse,multiverse/debian-installer,restricted,restricted/debian-installer,universe,universe/debian-installer" +dists="precise,precise-backports,precise-proposed,precise-updates,precise-security,trusty,trusty-backports,trusty-proposed,trusty-updates,trusty-security" +server="$1" +inPath="/ubuntu" +proto="rsync" +outpath="$SYNC_FILES" +rsyncOpt='-6 -aIL --partial' + +debmirror -h $server --no-check-gpg -a $arch -s $sections -d $dists -r $inPath -e $proto --rsync-options "$rsyncOpt" --verbose $outpath + +date --rfc-3339=seconds > "$SYNC_FILES/lastsync" +echo ">> Finished sync on $(date --rfc-3339=seconds)" + +# rm -f "$SYNC_LOCK" +exit 0