diff --git a/helpers/apt-download b/helpers/apt-download index 80bbaee..6d3a88b 100644 --- a/helpers/apt-download +++ b/helpers/apt-download @@ -2,21 +2,15 @@ set -e set -u set -o pipefail + +_here=`dirname $(realpath $0)` +if [[ -z ${LOADED_HELPERS:-} ]]; then + . ${_here}/helpers +fi + LOADED_APT_DOWNLOAD="yes" MAX_RETRY=${MAX_RETRY:-"3"} -function check-and-download() { - remote_file=$1 - local_file=$2 - echo "downloading ${remote_file}" - timeout -s INT 300 wget ${WGET_OPTIONS:-} -q -N -O ${local_file} ${remote_file} || { - rm ${local_file} - return 1 - } - return -} - - function apt-download-binary() { local base_url=$1 local dist=$2 diff --git a/helpers/helpers b/helpers/helpers new file mode 100644 index 0000000..b55d341 --- /dev/null +++ b/helpers/helpers @@ -0,0 +1,17 @@ +#!/bin/bash +set -e +set -u +set -o pipefail + +function check-and-download() { + remote_file=$1 + local_file=$2 + timeout -s INT 300 wget ${WGET_OPTIONS:-} -q -N -O ${local_file} ${remote_file} || { + rm ${local_file} + return 1 + } + return +} + +LOADED_HELPERS="yes" + diff --git a/helpers/tf-xml-filelist.py b/helpers/tf-xml-filelist.py new file mode 100755 index 0000000..d534900 --- /dev/null +++ b/helpers/tf-xml-filelist.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +import argparse +import xml.etree.ElementTree as ET + +def get_filelist(xmlstring): + r = ET.fromstring(xmlstring) + ns = { + 's3': 'http://doc.s3.amazonaws.com/2006-03-01', + } + filelist = [] + for cnt in r.findall('s3:Contents', ns): + key = cnt.find('s3:Key', ns) + fname = key.text + if fname.endswith('/') or not (fname.startswith('linux') or fname.startswith('mac')): + continue + + size = cnt.find('s3:Size', ns).text + filelist.append((fname, size)) + + return filelist + + +if __name__ == "__main__": + import fileinput + xmlstring = '\n'.join([line for line in fileinput.input()]) + filelist = get_filelist(xmlstring) + for fname, size in filelist: + print("{}\t{}".format(fname, size)) + +# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tensorflow.sh b/tensorflow.sh new file mode 100755 index 0000000..e330f7c --- /dev/null +++ b/tensorflow.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# requires: wget, python3 +set -u +set -e +set -o pipefail + +_here=`dirname $(realpath $0)` +. ${_here}/helpers/helpers + +XMLPARSE="${_here}/helpers/tf-xml-filelist.py" + +TF_UPSTREAM_BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://storage.googleapis.com/tensorflow"} +BASE_PATH="${TUNASYNC_WORKING_DIR}" + +# remove ending slash +BASE_PATH=${BASE_PATH%/} +TF_UPSTREAM_BASE_URL=${TF_UPSTREAM_BASE_URL%/} + +failed=0 +wget -O - "${TF_UPSTREAM_BASE_URL}/" | ${XMLPARSE} | while read -a tokens; do + filename=${tokens[0]} + filesize=${tokens[1]} + + # Notice: the filename starts with no leading '/'! + pkgurl="${TF_UPSTREAM_BASE_URL}/${filename}" + pkgdst="${BASE_PATH}/${filename}" + pkgdir=`dirname ${pkgdst}` + mkdir -p ${pkgdir} + + echo "downloading ${pkgurl}" + if [[ -z ${DRY_RUN:-} ]]; then + check-and-download ${pkgurl} ${pkgdst} || failed=1 + fi +done +exit $failed + + + + + + + + + +