mirror of
https://github.com/tuna/tunasync-scripts.git
synced 2025-04-20 12:42:50 +00:00
122 lines
2.8 KiB
Bash
Executable File
122 lines
2.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# requires: curl
|
|
set -e
|
|
set -o pipefail
|
|
|
|
function remove_broken() {
|
|
interval=$1
|
|
interval_file="${TUNASYNC_WORKING_DIR}/.tunasync_hackage_lastcheck"
|
|
now=`date +%s`
|
|
|
|
if [[ -f ${interval_file} ]]; then
|
|
lastcheck=`cat ${interval_file}`
|
|
((between = now - lastcheck))
|
|
if ((between < interval)); then
|
|
echo "skip checking"
|
|
return 0
|
|
fi
|
|
fi
|
|
echo "start checking"
|
|
|
|
mkdir -p "${TUNASYNC_WORKING_DIR}/package"
|
|
cd "${TUNASYNC_WORKING_DIR}/package"
|
|
|
|
for line in *; do
|
|
tar -tzf $line &>/dev/null || (printf 'FAIL %s\n' "$line"; rm $line) # && echo "OK"
|
|
done
|
|
|
|
echo `date +%s` > $interval_file
|
|
}
|
|
|
|
function must_download() {
|
|
src=$1
|
|
dst=$2
|
|
while true; do
|
|
echo "downloading: $dst"
|
|
curl -s -S --fail -L "$src" -o "$dst"
|
|
tar -tzf "$dst" >/dev/null || rm "$dst" && break
|
|
done
|
|
return 0
|
|
}
|
|
|
|
function hackage_mirror() {
|
|
local_pklist="/tmp/hackage_local_pklist_$$.list"
|
|
remote_pklist="/tmp/hackage_remote_pklist_$$.list"
|
|
base_url="https://hackage.haskell.org"
|
|
|
|
cd ${TUNASYNC_WORKING_DIR}
|
|
mkdir -p package
|
|
|
|
echo "Downloading index..."
|
|
rm index.tar.gz || true
|
|
curl -s -S --fail -L "${base_url}/01-index.tar.gz" -o index.tar.gz
|
|
|
|
# https://hackage.haskell.org/api#security
|
|
echo "Dowloading security features..."
|
|
jsons=("timestamp.json" "snapshot.json" "root.json" "mirrors.json")
|
|
for name in "${jsons[@]}"
|
|
do
|
|
curl -s -S --fail -L "${base_url}/${name}" -o "${name}"
|
|
done
|
|
|
|
echo "building local package list"
|
|
local tmp
|
|
tmp=(package/*)
|
|
tmp=(${tmp[@]#package/})
|
|
printf '%s\n' "${tmp[@]%.tar.gz}" | sort | uniq > "${local_pklist}"
|
|
|
|
echo "building remote package list"
|
|
tar -ztf index.tar.gz | (cut -d/ -f 1,2 2>/dev/null) | sed 's|/|-|' | sort | uniq > "${remote_pklist}"
|
|
|
|
echo "building download list"
|
|
# subtract local list from remote list
|
|
# this cannot use pipe, or the `wait` afterwards cannot wait
|
|
# because pipe spawns a subshell
|
|
while read pk; do
|
|
# ignore package suffix "preferred-versions"
|
|
# echo $pk
|
|
if [[ $pk = *-preferred-versions ]]; then
|
|
continue
|
|
fi
|
|
# limit concurrent level
|
|
bgcount=`jobs | wc -l`
|
|
while [[ $bgcount -ge 5 ]]; do
|
|
sleep 0.5
|
|
bgcount=`jobs | wc -l`
|
|
done
|
|
|
|
name="$pk.tar.gz"
|
|
if [ ! -e package/$name ]; then
|
|
must_download "${base_url}/package/$pk/$name" "package/$name" &
|
|
else
|
|
echo "skip existed: $name"
|
|
fi
|
|
done < <(comm "${remote_pklist}" "${local_pklist}" -23)
|
|
|
|
wait
|
|
|
|
# delete redundanty files
|
|
comm "$remote_pklist" "$local_pklist" -13 | while read pk; do
|
|
if [[ $pk == "preferred-versions" ]]; then
|
|
continue
|
|
fi
|
|
name="${pk}.tar.gz"
|
|
echo "deleting ${name}"
|
|
rm "package/$name"
|
|
done
|
|
|
|
cp index.tar.gz 01-index.tar.gz
|
|
ln -sf 01-index.tar.gz 00-index.tar.gz
|
|
}
|
|
|
|
function cleanup () {
|
|
echo "cleaning up"
|
|
[[ ! -z $local_pklist ]] && (rm $local_pklist $remote_pklist ; true)
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
remove_broken 86400
|
|
hackage_mirror
|
|
|
|
# vim: ts=4 sts=4 sw=4
|