tunasync-scripts/helpers/apt-download

#!/bin/bash
set -e
set -u
set -o pipefail
LOADED_APT_DOWNLOAD="yes"

function check-and-download() {
	remote_file=$1
	local_file=$2
	echo "downloading ${remote_file}"
	timeout -s INT 300 wget ${WGET_OPTIONS:-} -q -N -O ${local_file} ${remote_file} || {
		rm ${local_file}
		return 1
	}
	return
}


function apt-download-binary() {
	local base_url=$1
	local dist=$2
	local repo=$3
	local arch=$4
	local dest_base_dir=$5
	if [ -z $dest_base_dir ]; then
		echo "Destination directory is empty, cannot continue"
		return 1
	fi
	echo "Started mirroring ${base_url} ${dist}, ${repo}, ${arch}!"
	
	dist_dir="${dest_base_dir}/dists/${dist}"
	[ ! -d "$dist_dir" ] && mkdir -p "$dist_dir"
	dist_tmp_dir="${dist_dir}/.tmp"
	[ ! -d "$dist_tmp_dir" ] && mkdir -p "$dist_tmp_dir"
	rm -rf ${dist_tmp_dir}/*
	
	check-and-download "${base_url}/dists/${dist}/Contents-${arch}.gz" "${dist_tmp_dir}/Contents-${arch}.gz" || true
	check-and-download "${base_url}/dists/${dist}/InRelease"  "${dist_tmp_dir}/InRelease" || true
	check-and-download "${base_url}/dists/${dist}/Release" "${dist_tmp_dir}/Release" || {
		echo "Invalid Repository"
		return 1
	}
	check-and-download "${base_url}/dists/${dist}/Release.gpg" "${dist_tmp_dir}/Release.gpg" || true
	
	# Load Package Index URLs from Release file
	release_file="${dist_tmp_dir}/Release"

	pkgidx_dir="${dest_base_dir}/dists/${dist}/${repo}/binary-${arch}"
	pkgidx_tmp_dir="${pkgidx_dir}/.tmp"
	[ ! -d "$pkgidx_dir" ] && mkdir -p "$pkgidx_dir"
	[ ! -d "$pkgidx_tmp_dir" ] && mkdir -p "$pkgidx_tmp_dir"
	rm -rf ${pkgidx_tmp_dir}/*

	declare pkgidx_content=""
	declare cnt_start=false
	declare -i checksum_len
	if (grep -e '^SHA256:$' ${release_file} &>/dev/null); then
		checksum_cmd="sha256sum"; checksum_regex="^SHA256:$"; checksum_len=64
	elif (grep -e '^SHA1:$' ${release_file} &>/dev/null); then
		checksum_cmd="sha1sum"; checksum_regex="^SHA1:$"; checksum_len=40
	elif (grep -e '^MD5Sum:$' ${release_file} &>/dev/null); then
		checksum_cmd="md5sum"; checksum_regex="^MD5sum:$"; checksum_len=32
	fi

	while read line; do
		if [[ ${cnt_start} = true ]]; then
			read -a tokens <<< $line
			checksum=${tokens[0]}
			if [[ ${#checksum} != ${checksum_len} ]]; then
				break
			fi
			filesize=${tokens[1]}
			filename=${tokens[2]}
			if [[ "$filename" =~ ${repo}/binary-${arch} ]]; then
				# Load package list from Packages file
				pkgidx_filename=`basename $filename`
				pkgidx_file="${pkgidx_tmp_dir}/${pkgidx_filename}" 
				# dest_dir=`dirname ${pkgidx_file}`
				# [ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"

				pkglist_url="${base_url}/dists/${dist}/${filename}"
				check-and-download "${pkglist_url}" ${pkgidx_file} || {
					printf "Failed to download: %s\n" ${pkglist_url}
					return 1
				}
				echo "${checksum}  ${pkgidx_file}" | ${checksum_cmd} -c -
				if [ -z "${pkgidx_content}" -a -f ${pkgidx_file} ]; then
					echo "getting packages index content"
					case $filename in
						"*.bz2")
							pkgidx_content=`bunzip2 -c ${pkgidx_file}`
							;;
						"*.gz")
							pkgidx_content=`gunzip -c ${pkgidx_file}`
							;;
						*)
							pkgidx_content=`cat ${pkgidx_file}`
							;;
					esac
				fi
			fi
		else
			if [[ "$line" =~ ${checksum_regex} ]]; then
				cnt_start=true
			fi
		fi
	done < ${release_file}

	if [ -z "${pkgidx_content}" ]; then
		echo "index is empty, failed"
		return 1
	fi
	
	# Set checksum method
	if (echo -e "${pkgidx_content}" | grep -e '^SHA256' &>/dev/null); then
		checksum_cmd="sha256sum"; checksum_regex="^SHA256"
	elif (echo -e "${pkgidx_content}" | grep -e '^SHA1' &>/dev/null); then
		checksum_cmd="sha1sum"; checksum_regex="^SHA1" 
	elif (echo -e "${pkgidx_content}" | grep -e '^MD5sum' &>/dev/null); then
		checksum_cmd="md5sum"; checksum_regex="^MD5sum"
	fi

	# Download packages
	(echo -e "${pkgidx_content}" | grep -e '^Filename' -e '^Size' -e ${checksum_regex} | cut -d' ' -f 2) | \
	while read pkg_filename; read pkg_size; read pkg_checksum; do
		dest_filename="${dest_base_dir}/${pkg_filename}"
		dest_dir=`dirname ${dest_filename}`
		[ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"
		pkg_url="${base_url}/${pkg_filename}"
		declare downloaded=false
		if [ -f ${dest_filename} ]; then
			rsize=`stat -c "%s" ${dest_filename}`
			if [ ${rsize} -eq ${pkg_size} ]; then
				downloaded=true
				echo "Skipping ${pkg_filename}, size ${pkg_size}"
			fi
		fi
		while [ $downloaded != true ]; do
			echo "downloading ${pkg_url} to ${dest_filename}"
			if [[ -z ${APT_DRY_RUN:-} ]]; then
				wget ${WGET_OPTIONS:-} -q -O ${dest_filename} ${pkg_url} && {
					echo "${pkg_checksum}  ${dest_filename}" | ${checksum_cmd} -c - && downloaded=true # two space for md5sum/sha1sum/sha256sum check format
				}
			else
				downloaded=true
			fi
		done
	done

	mv ${pkgidx_tmp_dir}/* ${pkgidx_dir}
	mv ${dist_tmp_dir}/* ${dist_dir}

	rmdir ${pkgidx_tmp_dir} ${dist_tmp_dir}

	echo "Mirroring ${base_url} ${dist}, ${repo}, ${arch} done!"

}

# vim: ts=4 sts=4 sw=4