tunasync-scripts/helpers/apt-download

#!/bin/bash
set -e
set -u
set -o pipefail

_here=`dirname $(realpath ${BASH_SOURCE})`
if [[ -z ${LOADED_HELPERS:-} ]]; then
	. ${_here}/helpers
fi

LOADED_APT_DOWNLOAD="yes"
MAX_RETRY=${MAX_RETRY:-"3"}
DOWNLOAD_TIMEOUT=${DOWNLOAD_TIMEOUT:-"1800"}

function apt-download-binary() {
	local base_url=$1
	local dist=$2
	local repo=$3
	local arch=$4
	local dest_base_dir=$5
	local filelist="${6:-"/dev/null"}"

	if [ -z $dest_base_dir ]; then
		echo "Destination directory is empty, cannot continue"
		return 1
	fi
	echo "Started mirroring ${base_url} ${dist}, ${repo}, ${arch}!"
	
	dist_dir="${dest_base_dir}/dists/${dist}"
	[ ! -d "$dist_dir" ] && mkdir -p "$dist_dir"
	dist_tmp_dir="${dist_dir}/.tmp"
	[ ! -d "$dist_tmp_dir" ] && mkdir -p "$dist_tmp_dir"
	rm -rf ${dist_tmp_dir}/*
	
	check-and-download "${base_url}/dists/${dist}/Contents-${arch}.gz" "${dist_tmp_dir}/Contents-${arch}.gz" || true
	check-and-download "${base_url}/dists/${dist}/InRelease"  "${dist_tmp_dir}/InRelease" || true
	check-and-download "${base_url}/dists/${dist}/Release" "${dist_tmp_dir}/Release" || {
		echo "Invalid Repository"
		return 1
	}
	check-and-download "${base_url}/dists/${dist}/Release.gpg" "${dist_tmp_dir}/Release.gpg" || true
	
	# download Contents file
	comp_dir="${dest_base_dir}/dists/${dist}/${repo}"
	comp_tmp_dir="${comp_dir}/.tmp"
	[ ! -d "${comp_tmp_dir}" ] && mkdir -p "${comp_tmp_dir}"
	rm -rf ${comp_tmp_dir}/*
	check-and-download "${base_url}/dists/${dist}/${repo}/Contents-${arch}" "${comp_tmp_dir}/Contents-${arch}" || true
	check-and-download "${base_url}/dists/${dist}/${repo}/Contents-${arch}.gz" "${comp_tmp_dir}/Contents-${arch}.gz" || true
	check-and-download "${base_url}/dists/${dist}/${repo}/Contents-${arch}.bz2" "${comp_tmp_dir}/Contents-${arch}.bz2" || true

	
	# Load Package Index URLs from Release file
	release_file="${dist_tmp_dir}/Release"

	pkgidx_dir="${dest_base_dir}/dists/${dist}/${repo}/binary-${arch}"
	pkgidx_tmp_dir="${pkgidx_dir}/.tmp"
	[ ! -d "$pkgidx_dir" ] && mkdir -p "$pkgidx_dir"
	[ ! -d "$pkgidx_tmp_dir" ] && mkdir -p "$pkgidx_tmp_dir"
	rm -rf ${pkgidx_tmp_dir}/*

	declare pkgidx_content=""
	declare cnt_start=false
	declare -i checksum_len
	if (grep -e '^SHA256:$' ${release_file} &>/dev/null); then
		checksum_cmd="sha256sum"; checksum_regex="^SHA256:$"; checksum_len=64
	elif (grep -e '^SHA1:$' ${release_file} &>/dev/null); then
		checksum_cmd="sha1sum"; checksum_regex="^SHA1:$"; checksum_len=40
	elif (grep -e '^MD5Sum:$' ${release_file} &>/dev/null); then
		checksum_cmd="md5sum"; checksum_regex="^MD5sum:$"; checksum_len=32
	fi

	while read line; do
		if [[ ${cnt_start} = true ]]; then
			read -a tokens <<< $line
			checksum=${tokens[0]}
			if [[ ${#checksum} != ${checksum_len} ]]; then
				break
			fi
			filesize=${tokens[1]}
			filename=${tokens[2]}
			if [[ "$filename" =~ ^${repo}/binary-${arch} ]]; then
				# Load package list from Packages file
				pkgidx_filename=`basename $filename`
				pkgidx_file="${pkgidx_tmp_dir}/${pkgidx_filename}" 
				# dest_dir=`dirname ${pkgidx_file}`
				# [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"

				pkglist_url="${base_url}/dists/${dist}/${filename}"
				check-and-download "${pkglist_url}" ${pkgidx_file} || {
					printf "Failed to download: %s\n" ${pkglist_url}
					return 1
				}
				echo "${checksum}  ${pkgidx_file}" | ${checksum_cmd} -c -
				if [ -z "${pkgidx_content}" -a -f ${pkgidx_file} ]; then
					echo "getting packages index content"
					case $filename in
						"*.bz2")
							pkgidx_content=`bunzip2 -c ${pkgidx_file}`
							;;
						"*.gz")
							pkgidx_content=`gunzip -c ${pkgidx_file}`
							;;
						*)
							pkgidx_content=`cat ${pkgidx_file}`
							;;
					esac
				fi
			fi
		else
			if [[ "$line" =~ ${checksum_regex} ]]; then
				cnt_start=true
			fi
		fi
	done < ${release_file}

	if [ -z "${pkgidx_content}" ]; then
		echo "index is empty, failed"
		return 1
	fi
	
	# Set checksum method
	if (echo -e "${pkgidx_content}" | grep -e '^SHA256' &>/dev/null); then
		checksum_cmd="sha256sum"; checksum_regex="^SHA256"
	elif (echo -e "${pkgidx_content}" | grep -e '^SHA1' &>/dev/null); then
		checksum_cmd="sha1sum"; checksum_regex="^SHA1" 
	elif (echo -e "${pkgidx_content}" | grep -e '^MD5sum' &>/dev/null); then
		checksum_cmd="md5sum"; checksum_regex="^MD5sum"
	fi
	
	ERROR=0

	awk_script='BEGIN{FS="\n";RS="\n\n"}{for(i=1;i<=NF;i++){if($i ~ /^Filename/){fn=$i;gsub(/^.+: /,"",fn)}else if($i ~ /^Size/){sz=$i;gsub(/^.+: /,"",sz)}else if($i ~ /'
	awk_script+="${checksum_regex}"
	awk_script+='/){hash=$i;gsub(/^.+: /,"",hash)}}print fn;print sz;print hash }'

	# Download packages
	(echo -e "${pkgidx_content}" | awk "$awk_script") | \
	while read pkg_filename; read pkg_size; read pkg_checksum; do
		echo ${pkg_filename} >> $filelist
		dest_filename="${dest_base_dir}/${pkg_filename}"
		dest_dir=`dirname ${dest_filename}`
		[ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"
		pkg_url="${base_url}/${pkg_filename}"
		declare downloaded=false
		if [ -f ${dest_filename} ]; then
			rsize=`stat -c "%s" ${dest_filename}`
			if [ ${rsize} -eq ${pkg_size} ]; then
				downloaded=true
				echo "Skipping ${pkg_filename}, size ${pkg_size}"
			fi
		fi
		[[ $downloaded == true ]] && continue

		for retry in `seq ${MAX_RETRY}`; do
			echo "downloading ${pkg_url} to ${dest_filename}"
			if [[ -z ${APT_DRY_RUN:-} ]]; then
				timeout -s INT "$DOWNLOAD_TIMEOUT" wget ${WGET_OPTIONS:-} -q -O ${dest_filename} ${pkg_url} && {
					# two space for md5sum/sha1sum/sha256sum check format
					echo "${pkg_checksum}  ${dest_filename}" | ${checksum_cmd} -c - && downloaded=true
				}
			else
				downloaded=true
			fi
			[[ $downloaded == true ]] && break
		done
		[[ $downloaded == false ]] && ERROR=1
	done
	
	dir-not-empty ${pkgidx_tmp_dir} && mv ${pkgidx_tmp_dir}/* ${pkgidx_dir}
	dir-not-empty ${dist_tmp_dir} && mv ${dist_tmp_dir}/* ${dist_dir}
	dir-not-empty ${comp_tmp_dir} && mv ${comp_tmp_dir}/* ${comp_dir}

	rmdir ${pkgidx_tmp_dir} ${dist_tmp_dir} ${comp_tmp_dir}

	echo "Mirroring ${base_url} ${dist}, ${repo}, ${arch} done!"

	return $ERROR
}

function apt-delete-old-debs {
	local base_dir=$1
	local remote_filelist=$2

	[[ ! -d ${base_dir} ]] && return 1
	[[ ! -f ${remote_filelist} ]] && return 1
	
	local_filelist="${base_dir}/filelist.local"
	(cd ${base_dir}; find . -type f -iname "*.deb") | sed 's+^\./++' > ${local_filelist}
	comm <(sort $remote_filelist) <(sort $local_filelist) -13 | while read file; do
		echo "deleting ${base_dir}/${file}"
		rm "${base_dir}/${file}"
	done
	rm $local_filelist
}

# vim: ts=4 sts=4 sw=4