tensorflow script

Signed-off-by: bigeagle <justin.w.xd@gmail.com>

helpers/apt-download

@@ -2,21 +2,15 @@
 set -e
 set -u
 set -o pipefail
+
+_here=`dirname $(realpath $0)`
+if [[ -z ${LOADED_HELPERS:-} ]]; then
+	. ${_here}/helpers
+fi
+
 LOADED_APT_DOWNLOAD="yes"
 MAX_RETRY=${MAX_RETRY:-"3"}
 
-function check-and-download() {
-	remote_file=$1
-	local_file=$2
-	echo "downloading ${remote_file}"
-	timeout -s INT 300 wget ${WGET_OPTIONS:-} -q -N -O ${local_file} ${remote_file} || {
-		rm ${local_file}
-		return 1
-	}
-	return
-}
-
-
 function apt-download-binary() {
 	local base_url=$1
 	local dist=$2

helpers/helpers

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+set -u
+set -o pipefail
+
+function check-and-download() {
+	remote_file=$1
+	local_file=$2
+	timeout -s INT 300 wget ${WGET_OPTIONS:-} -q -N -O ${local_file} ${remote_file} || {
+		rm ${local_file}
+		return 1
+	}
+	return
+}
+
+LOADED_HELPERS="yes"
+

helpers/tf-xml-filelist.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+import argparse
+import xml.etree.ElementTree as ET
+
+def get_filelist(xmlstring):
+    r = ET.fromstring(xmlstring)
+    ns = {
+        's3': 'http://doc.s3.amazonaws.com/2006-03-01',
+    }
+    filelist = []
+    for cnt in r.findall('s3:Contents', ns):
+        key = cnt.find('s3:Key', ns)
+        fname = key.text
+        if fname.endswith('/') or not (fname.startswith('linux') or fname.startswith('mac')):
+            continue
+
+        size = cnt.find('s3:Size', ns).text
+        filelist.append((fname, size))
+
+    return filelist
+
+
+if __name__ == "__main__":
+    import fileinput
+    xmlstring = '\n'.join([line for line in fileinput.input()])
+    filelist = get_filelist(xmlstring)
+    for fname, size in filelist:
+        print("{}\t{}".format(fname, size))
+
+# vim: ts=4 sw=4 sts=4 expandtab

tensorflow.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# requires: wget, python3
+set -u
+set -e
+set -o pipefail
+
+_here=`dirname $(realpath $0)`
+. ${_here}/helpers/helpers
+
+XMLPARSE="${_here}/helpers/tf-xml-filelist.py"
+
+TF_UPSTREAM_BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://storage.googleapis.com/tensorflow"}
+BASE_PATH="${TUNASYNC_WORKING_DIR}"
+
+# remove ending slash
+BASE_PATH=${BASE_PATH%/}
+TF_UPSTREAM_BASE_URL=${TF_UPSTREAM_BASE_URL%/}
+
+failed=0
+wget -O - "${TF_UPSTREAM_BASE_URL}/" | ${XMLPARSE} | while read -a tokens; do
+	filename=${tokens[0]}
+	filesize=${tokens[1]}
+	
+	# Notice: the filename starts with no leading '/'!
+	pkgurl="${TF_UPSTREAM_BASE_URL}/${filename}"
+	pkgdst="${BASE_PATH}/${filename}"
+	pkgdir=`dirname ${pkgdst}`
+	mkdir -p ${pkgdir}
+	
+	echo "downloading ${pkgurl}"
+	if [[ -z ${DRY_RUN:-} ]]; then
+		check-and-download ${pkgurl} ${pkgdst} || failed=1
+	fi
+done
+exit $failed
+
+
+
+
+
+
+
+
+
+