yacloud/tunasync-scripts
1
0
Fork 0
mirror of https://github.com/tuna/tunasync-scripts.git synced 2025-04-20 12:42:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

reimplement the lxc-images.sh to solve tuna/issues#558

Browse Source 
the method described in https://github.com/tuna/issues/issues/558#issuecomment-589921950
This commit is contained in:
Yuxiang Zhang 2020-02-22 17:20:31 +08:00 committed by GitHub
parent 3c3c178dcb
commit 38a46e7e29
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 52 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
lxc-images.sh
View File

@ -1,29 +1,71 @@
#!/bin/bash #!/bin/bash
# requires: lftp wget jq # requires: lftp wget jq
set -e
set -o pipefail
BASE_URL="${TUNASYNC_UPSTREAM_URL:-"http://images.linuxcontainers.org/"}" BASE_URL="${TUNASYNC_UPSTREAM_URL:-"http://images.linuxcontainers.org"}"
function sync_lxc_images() { function sync_lxc_images() {
repo_url="$1" repo_url="$1"
repo_dir="$2" repo_dir="$2"
[ ! -d "$repo_dir" ] && mkdir -p "$repo_dir" [[ ! -d "$repo_dir" ]] && mkdir -p "$repo_dir"
cd $repo_dir cd "$repo_dir"
lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye" lftp "${repo_url}" -e "mirror --verbose -P 5 --delete --only-newer; bye"
echo "lftp returns $?"
} }
sync_lxc_images "${BASE_URL}/images" "${TUNASYNC_WORKING_DIR}/images"
echo "=== Downloading /meta/1.0 ==="
mkdir -p "${TUNASYNC_WORKING_DIR}/meta/1.0" mkdir -p "${TUNASYNC_WORKING_DIR}/meta/1.0"
for i in index-system index-system.asc index-user index-user.asc; do for i in index-system index-system.asc index-user index-user.asc; do
wget -O "${TUNASYNC_WORKING_DIR}/meta/1.0/"$i "${BASE_URL}/meta/1.0/"$i wget -O "${TUNASYNC_WORKING_DIR}/meta/1.0/$i.work-in-progress" "${BASE_URL}/meta/1.0/$i"
done done
echo "=== Downloading /streams/v1 ==="
mkdir -p "${TUNASYNC_WORKING_DIR}/streams/v1" mkdir -p "${TUNASYNC_WORKING_DIR}/streams/v1"
wget -O "${TUNASYNC_WORKING_DIR}/streams/v1/index.json" "${BASE_URL}/streams/v1/index.json" wget -O "${TUNASYNC_WORKING_DIR}/streams/v1/index.json.work-in-progress" "${BASE_URL}/streams/v1/index.json"
jq -r '.index[].path' "${TUNASYNC_WORKING_DIR}/streams/v1/index.json" | while read line; do jq -r '.index[].path' "${TUNASYNC_WORKING_DIR}/streams/v1/index.json.work-in-progress" | while read line; do
[ ! -d "${TUNASYNC_WORKING_DIR}/$(dirname $line)" ] && mkdir -p "${TUNASYNC_WORKING_DIR}/$(dirname $line)" [[ ! -d "${TUNASYNC_WORKING_DIR}/$(dirname $line)" ]] && mkdir -p "${TUNASYNC_WORKING_DIR}/$(dirname $line)"
wget -O "${TUNASYNC_WORKING_DIR}/${line}" "${BASE_URL}/${line}" wget -O "${TUNASYNC_WORKING_DIR}/${line}.work-in-progress" "${BASE_URL}/${line}"
done
echo "=== Downloading images ==="
images_json="${TUNASYNC_WORKING_DIR}/streams/v1/images.json.work-in-progress"
[[ -f "$images_json" ]] || exit 1
jq -r '.products[].versions[].items[].path' "$images_json" > /tmp/filelist.txt
#cut -f 6 -d ';' "${TUNASYNC_WORKING_DIR}/meta/1.0/index-system.work-in-progress"
cat /tmp/filelist.txt | while read line; do
# $line looks like 'images/ubuntu/xenial/armhf/default/20200219_07:42/rootfs.tar.xz'
dir="$(dirname $line)"
if [[ "$dir" = "$last_dir" ]]; then
continue
fi
last_dir="$dir"
echo "=== Syncing $dir ==="
sync_lxc_images "${BASE_URL}/${dir}" "${TUNASYNC_WORKING_DIR}/${dir}"
done
echo "=== Replacing /meta/1.0 ==="
for i in index-system index-system.asc index-user index-user.asc; do
mv -f "${TUNASYNC_WORKING_DIR}/meta/1.0/$i.work-in-progress" "${TUNASYNC_WORKING_DIR}/meta/1.0/$i"
done
echo "=== Replacing /streams/v1 ==="
jq -r '.index[].path' "${TUNASYNC_WORKING_DIR}/streams/v1/index.json.work-in-progress" | while read line; do
mv -f "${TUNASYNC_WORKING_DIR}/${line}.work-in-progress" "${TUNASYNC_WORKING_DIR}/${line}"
done
mv -f "${TUNASYNC_WORKING_DIR}/streams/v1/index.json.work-in-progress" "${TUNASYNC_WORKING_DIR}/streams/v1/index.json"
echo "=== Removing old images ==="
cd "${TUNASYNC_WORKING_DIR}"
find images/ -maxdepth 5 -mindepth 5 -mtime +3 | while read line; do
# $line looks like 'images/ubuntu/xenial/armhf/default/20200217_07:42'
grep --quiet "$line" /tmp/filelist.txt || ( echo "Removing $line"; rm -rf "$line" )
done done