From 8b3b61867bae13874325900b3fe049d1ac76061a Mon Sep 17 00:00:00 2001 From: bigeagle Date: Thu, 22 Dec 2016 22:17:42 +0800 Subject: [PATCH] generate tensorflow index --- helpers/tf-gen-index.py | 80 ++++++++++++++++++ tensorflow.sh | 14 +--- test/tf.filelist | 178 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 262 insertions(+), 10 deletions(-) create mode 100755 helpers/tf-gen-index.py create mode 100644 test/tf.filelist diff --git a/helpers/tf-gen-index.py b/helpers/tf-gen-index.py new file mode 100755 index 0000000..9c528cc --- /dev/null +++ b/helpers/tf-gen-index.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +import json +import re + + +def version(v: str): + return tuple(map(int, v.split('.'))) + + +def generate_fileindex(filelist): + fname_re = re.compile( + r''' + (?P[a-z_]+?)- # match tensorflow and tensorflow_gpu + (?P[\d.]+?)- # match version + (?P[cpy\d]+?)- # python version, 'rc's are ignored + (?P.+?\.whl) # everything else + ''', + re.VERBOSE, + ) + + versions = set([]) + pythons = { + 'linux': set([]), + 'mac': set([]), + } + pkglist = [] + + min_ver = version('0.10.0') + + for fpath in filelist: + tokens = fpath.split('/') + if len(tokens) != 3: + continue + os, xpu, fname = tokens + m = fname_re.match(fname) + if m is None: + continue + + if not m.group('tensorflow').startswith('tensorflow'): + continue + + tfver = m.group('tfver') + if version(tfver) < min_ver: + continue + versions.add(tfver) + + pyver = m.group('python') + pythons[os].add(pyver) + + pkglist.append({ + 'os': os, + 'xpu': xpu, + 'python': pyver, + 'version': tfver, + 'filename': fname, + }) + + pythons = { + k: sorted(list(v), reverse=True) + for k, v in pythons.items() + } + versions = sorted(list(versions), key=lambda x: version(x), reverse=True) + + index = { + 'versions': versions, + 'pythons': pythons, + 'pkglist': pkglist, + } + + return index + + +if __name__ == "__main__": + import fileinput + filelist = [line.strip() for line in fileinput.input()] + index = generate_fileindex(filelist) + print(json.dumps(index, sort_keys=True, indent=2)) + + +# vim: ts=4 sw=4 sts=4 expandtab diff --git a/tensorflow.sh b/tensorflow.sh index 71b3012..536ab12 100755 --- a/tensorflow.sh +++ b/tensorflow.sh @@ -6,6 +6,7 @@ set -o pipefail _here=`dirname $(realpath $0)` XMLPARSE="${_here}/helpers/tf-xml-filelist.py" +INDEXGEN="${_here}/helpers/tf-gen-index.py" TF_UPSTREAM_BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://storage.googleapis.com/tensorflow"} BASE_PATH="${TUNASYNC_WORKING_DIR}" @@ -41,14 +42,7 @@ wget -O - "${TF_UPSTREAM_BASE_URL}/" | ${XMLPARSE} | while read -a tokens; do fi done +find ${BASE_PATH} -type f -name '*.whl' -printf '%P\n' | \ + ${INDEXGEN} > "${BASE_PATH}/releases.json" + exit $failed - - - - - - - - - - diff --git a/test/tf.filelist b/test/tf.filelist new file mode 100644 index 0000000..9db5476 --- /dev/null +++ b/test/tf.filelist @@ -0,0 +1,178 @@ +linux/gpu/tensorflow-0.10.0-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow-0.10.0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow_gpu-0.12.0rc0-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow_gpu-0.12.0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0rc0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow-0.10.0rc0-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow-0.5.0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.9.0rc0-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.8.0rc0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow-0.6.0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow_gpu-0.12.0rc1-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.7.1-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow-0.8.0rc0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.9.0rc0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.9.0rc0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow-0.6.0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.10.0rc0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow_gpu-0.12.0rc0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.10.0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0rc0-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow_gpu-0.12.0-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow_gpu-0.12.0rc0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow_gpu-0.12.0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.6.0-cp34-none-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow-0.7.1-cp34-none-linux_x86_64.whl +linux/gpu/tensorflow_gpu-0.12.0rc1-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow-0.7.0-py2-none-linux_x86_64.whl +linux/gpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow_gpu-0.12.0rc1-cp35-cp35m-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.11.0rc0-cp27-none-linux_x86_64.whl +linux/gpu/tensorflow-0.7.0-py3-none-linux_x86_64.whl +linux/gpu/tensorflow-0.10.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.8.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.6.0-cp34-none-linux_x86_64.whl +linux/cpu/tensorflow-0.12.0-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.7.0-py2-none-linux_x86_64.whl +linux/cpu/protobuf-3.0.0-cp35-cp35m-linux_x86_64.whl +linux/cpu/tensorflow-0.7.1-cp34-none-linux_x86_64.whl +linux/cpu/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.9.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/protobuf-3.0.0b2.post2-cp34-none-linux_x86_64.whl +linux/cpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl +linux/cpu/tensorflow-0.10.0rc0-cp35-cp35m-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0rc0-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.12.0rc1-cp35-cp35m-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl +linux/cpu/tensorflow-0.12.0rc1-cp34-cp34m-linux_x86_64.whl +linux/cpu/protobuf-3.0.0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0rc0-cp35-cp35m-linux_x86_64.whl +linux/cpu/tensorflow-0.12.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.7.0-py3-none-linux_x86_64.whl +linux/cpu/cloudml/tensorflow-0.11.0-cp27-none-linux_x86_64.whl +linux/cpu/cloudml/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl +linux/cpu/cloudml/tensorflow-0.12.0rc1-cp27-none-linux_x86_64.whl +linux/cpu/cloudml/tensorflow-0.12.0-cp27-none-linux_x86_64.whl +linux/cpu/cloudml/tensorflow-0.11.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/cloudml/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl +linux/cpu/cloudml/tensorflow-0.12.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.10.0rc0-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl +linux/cpu/protobuf-3.0.0-cp3-none-linux_x86_64.whl +linux/cpu/protobuf-3.0.0-cp34-cp34m-linux_x86_64.whl +linux/cpu/protobuf-3.1.0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.10.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/protobuf-3.0.0b3-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.12.0-cp35-cp35m-linux_x86_64.whl +linux/cpu/tensorflow-0.12.0rc1-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.10.0-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.5.0-cp27-none-linux_x86_64.whl +linux/cpu/protobuf-3.0.0-cp27-cp27mu-linux_x86_64.whl +linux/cpu/tensorflow-0.12.0rc0-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl +linux/cpu/tensorflow-0.10.0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.9.0rc0-cp35-cp35m-linux_x86_64.whl +linux/cpu/tensorflow-0.6.0-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0-cp35-cp35m-linux_x86_64.whl +linux/cpu/protobuf-3.0.0b3-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.8.0rc0-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0-cp34-cp34m-linux_x86_64.whl +linux/cpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.12.0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.9.0rc0-cp34-cp34m-linux_x86_64.whl +linux/cpu/protobuf-3.0.0b2.post2-cp27-none-linux_x86_64.whl +linux/cpu/protobuf-3.1.0-cp35-none-linux_x86_64.whl +linux/cpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.7.1-cp27-none-linux_x86_64.whl +linux/cpu/protobuf-3.1.0-cp27-cp27mu-linux_x86_64.whl +linux/cpu/tensorflow-0.10.0-cp35-cp35m-linux_x86_64.whl +linux/cpu/tensorflow-0.6.0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.12.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.11.0-cp27-none-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.10.0-cp27-none-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.11.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.9.0-cp27-none-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.12.0rc1-cp27-none-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.10.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.12.0-cp27-none-linux_x86_64.whl +linux/cpu/debian/jessie/tensorflow-0.9.0rc0-cp27-none-linux_x86_64.whl +linux/cpu/tensorflow-0.12.0rc0-cp35-cp35m-linux_x86_64.whl +mac/tensorflow-0.6.0-py2-none-any.whl +mac/tensorflow-0.6.0-py3-none-any.whl +mac/tensorflow-0.8.0rc0-py2-none-any.whl +mac/tensorflow-0.8.0rc0-py3-none-any.whl +mac/tensorflow-0.7.1-cp35-none-any.whl +mac/tensorflow-0.8.0-py3-none-any.whl +mac/tensorflow-0.8.0-py2-none-any.whl +mac/tensorflow-0.9.0rc0-py2-none-any.whl +mac/cpu/tensorflow-0.12.0-py2-none-any.whl +mac/cpu/tensorflow-0.12.0-py3-none-any.whl +mac/cpu/tensorflow-0.10.0rc0-py3-none-any.whl +mac/cpu/tensorflow-0.10.0rc0-py2-none-any.whl +mac/cpu/tensorflow-0.10.0-py2-none-any.whl +mac/cpu/tensorflow-0.11.0rc2-py2-none-any.whl +mac/cpu/tensorflow-0.11.0rc2-py3-none-any.whl +mac/cpu/tensorflow-0.10.0-py3-none-any.whl +mac/cpu/tensorflow-0.11.0rc0-py3-none-any.whl +mac/cpu/tensorflow-0.11.0rc0-py2-none-any.whl +mac/cpu/protobuf-3.0.0-cp3-cp3m-macosx_10_11_x86_64.whl +mac/cpu/tensorflow-0.11.0rc1-py3-none-any.whl +mac/cpu/tensorflow-0.11.0rc1-py2-none-any.whl +mac/cpu/tensorflow-0.12.0rc0-py2-none-any.whl +mac/cpu/protobuf-3.0.0-cp35-cp35m-macosx_10_11_x86_64.whl +mac/cpu/tensorflow-0.11.0-py3-none-any.whl +mac/cpu/tensorflow-0.11.0-py2-none-any.whl +mac/cpu/tensorflow-0.12.0rc0-py3-none-any.whl +mac/cpu/tensorflow-0.12.0rc1-py2-none-any.whl +mac/cpu/tensorflow-0.12.0rc1-py3-none-any.whl +mac/cpu/protobuf-3.0.0-cp27-cp27m-macosx_10_11_x86_64.whl +mac/cpu/protobuf-3.1.0-cp27-none-macosx_10_11_x86_64.whl +mac/cpu/protobuf-3.1.0-cp35-none-macosx_10_11_x86_64.whl +mac/protobuf-3.0.0b2.post2-cp35-none-any.whl +mac/tensorflow-0.9.0rc0-py3-none-any.whl +mac/gpu/tensorflow_gpu-0.12.0rc1-py3-none-any.whl +mac/gpu/tensorflow_gpu-0.12.0rc1-py2-none-any.whl +mac/gpu/tensorflow_gpu-0.12.0rc0-py3-none-any.whl +mac/gpu/tensorflow-0.10.0rc0-py3-none-any.whl +mac/gpu/tensorflow_gpu-0.12.0rc0-py2-none-any.whl +mac/gpu/tensorflow-0.10.0rc0-py2-none-any.whl +mac/gpu/tensorflow-0.11.0rc1-py3-none-any.whl +mac/gpu/tensorflow-0.10.0-py3-none-any.whl +mac/gpu/tensorflow-0.10.0-py2-none-any.whl +mac/gpu/tensorflow-0.11.0rc1-py2-none-any.whl +mac/gpu/tensorflow-0.11.0rc0-py3-none-any.whl +mac/gpu/tensorflow-0.11.0rc0-py2-none-any.whl +mac/gpu/tensorflow-0.11.0rc2-py2-none-any.whl +mac/gpu/tensorflow-0.11.0rc2-py3-none-any.whl +mac/gpu/tensorflow-0.11.0-py2-none-any.whl +mac/gpu/tensorflow-0.11.0-py3-none-any.whl +mac/gpu/tensorflow_gpu-0.12.0-py3-none-any.whl +mac/gpu/tensorflow_gpu-0.12.0-py2-none-any.whl +mac/protobuf-3.0.0b2.post2-cp27-none-any.whl +mac/tensorflow-0.7.1-cp27-none-any.whl +mac/tensorflow-0.5.0-py2-none-any.whl +mac/tensorflow-0.9.0-py2-none-any.whl +mac/tensorflow-0.9.0-py3-none-any.whl +mac/tensorflow-0.7.0-py3-none-any.whl +mac/tensorflow-0.7.0-py2-none-any.whl