update hackage.sh

scripts/hackage.sh

@@ -1,38 +1,85 @@
 #!/bin/bash
+set -e
 
 function remove_broken() {
-	working_dir=$1
-	cd $working_dir
-	mkdir -p package
-	filelist="/tmp/hackage_filelist_$$.txt"
-	brokenlist="/tmp/hackage_brokenlist_$$.txt"
-	ls > ${filelist}
-	touch $brokenlist
-	while read line; do echo $line ; tar -tzf $line >/dev/null || echo $line >>$brokenlist; done <$filelist
-	cat $brokenlist | xargs rm
+	interval=$1
+	interval_file="/tmp/hackage_lastcheck"
+	now=`date +%s`
 
-	rm $brokenlist
+	if [[ -f ${interval_file} ]]; then
+		lastcheck=`cat ${interval_file}`
+		between=$(echo "${now}-${lastcheck}" | bc)
+		[[ $between -lt $interval ]] && echo "skip checking"; return 0
+	fi
+	echo "start checking"
+
+	mkdir -p "${TUNASYNC_WORKING_DIR}/package"
+	cd "${TUNASYNC_WORKING_DIR}/package"
+
+	ls | while read line; do 
+		echo -n "$line\t\t"
+		tar -tzf $line >/dev/null || (echo "FAIL"; rm $line) && echo "OK"
+	done
+	
+	echo `date +%s` > $interval_file
+}
+
+function must_download() {
+	src=$1
+	dst=$2
+	while true; do
+		echo "downloading: $name"
+		wget "$src" -O "$dst" &>/dev/null || true
+		tar -tzf package/$name >/dev/null || rm package/$name && break 
+	done
 }
 
 function hackage_mirror() {
-	working_dir=$1
-	cd $working_dir
-	# echo "Cleaning up..."
-	# rm 00-index.tar.gz
+	local_pklist="/tmp/hackage_local_pklist_$$.list"
+	remote_pklist="/tmp/hackage_remote_pklist_$$.list"
+	
+	cd ${TUNASYNC_WORKING_DIR}
 	mkdir -p package
+
 	echo "Downloading index..."
-	rm index.tar.gz
-	axel http://hdiff.luite.com/packages/archive/index.tar.gz -o index.tar.gz
-	for splitpk in `tar ztf index.tar.gz | cut -d/ -f 1,2 2>/dev/null`; do
-		pk=`echo $splitpk | sed 's|/|-|'`
-		name=$pk.tar.gz
+	rm index.tar.gz || true
+	axel http://hdiff.luite.com/packages/archive/index.tar.gz -o index.tar.gz > /dev/null
+	
+	echo "building local package list"
+	ls package | sed "s/\.tar\.gz$//" > $local_pklist
+	echo "preferred-versions" >> $local_pklist  # ignore preferred-versions
+	
+	echo "building remote package list"
+	tar ztf index.tar.gz | (cut -d/ -f 1,2 2>/dev/null) | sed 's|/|-|' > $remote_pklist
+	
+	echo "building download list"
+	# substract local list from remote list
+	comm <(sort $remote_pklist) <(sort $local_pklist) -3 | while read pk; do
+		# limit concurrent level
+		bgcount=`jobs | wc -l`
+		while [[ $bgcount -ge 5 ]]; do
+			sleep 0.5
+			bgcount=`jobs | wc -l`
+		done
+		
+		name="$pk.tar.gz"
 		if [[ ! -a package/$name ]]; then
-			axel http://hackage.haskell.org/package/$pk/$name -o package/$name
+			must_download "http://hackage.haskell.org/package/$pk/$name" "package/$name" &
+		else
+			echo "skip existed: $name"
 		fi
 	done
-	rm package/preferred-versions.tar.gz
+
 	cp index.tar.gz 00-index.tar.gz
 }
 
-remove_broken "${TUNASYNC_WORKING_DIR}/"
-hackage_mirror "${TUNASYNC_WORKING_DIR}/"
+function cleanup () {
+	echo "cleaning up"
+	[[ ! -z $local_pklist ]] && (rm $local_pklist $remote_pklist ; true)
+}
+
+trap cleanup EXIT
+remove_broken 86400
+hackage_mirror 
+
+# vim: ts=4 sts=4 sw=4