generate tensorflow index

This commit is contained in:
bigeagle 2016-12-22 22:17:42 +08:00
parent cf0375b4f9
commit 8b3b61867b
3 changed files with 262 additions and 10 deletions

80
helpers/tf-gen-index.py Executable file
View File

@ -0,0 +1,80 @@
#!/usr/bin/env python3
import json
import re
def version(v: str):
return tuple(map(int, v.split('.')))
def generate_fileindex(filelist):
fname_re = re.compile(
r'''
(?P<tensorflow>[a-z_]+?)- # match tensorflow and tensorflow_gpu
(?P<tfver>[\d.]+?)- # match version
(?P<python>[cpy\d]+?)- # python version, 'rc's are ignored
(?P<rest>.+?\.whl) # everything else
''',
re.VERBOSE,
)
versions = set([])
pythons = {
'linux': set([]),
'mac': set([]),
}
pkglist = []
min_ver = version('0.10.0')
for fpath in filelist:
tokens = fpath.split('/')
if len(tokens) != 3:
continue
os, xpu, fname = tokens
m = fname_re.match(fname)
if m is None:
continue
if not m.group('tensorflow').startswith('tensorflow'):
continue
tfver = m.group('tfver')
if version(tfver) < min_ver:
continue
versions.add(tfver)
pyver = m.group('python')
pythons[os].add(pyver)
pkglist.append({
'os': os,
'xpu': xpu,
'python': pyver,
'version': tfver,
'filename': fname,
})
pythons = {
k: sorted(list(v), reverse=True)
for k, v in pythons.items()
}
versions = sorted(list(versions), key=lambda x: version(x), reverse=True)
index = {
'versions': versions,
'pythons': pythons,
'pkglist': pkglist,
}
return index
if __name__ == "__main__":
import fileinput
filelist = [line.strip() for line in fileinput.input()]
index = generate_fileindex(filelist)
print(json.dumps(index, sort_keys=True, indent=2))
# vim: ts=4 sw=4 sts=4 expandtab

View File

@ -6,6 +6,7 @@ set -o pipefail
_here=`dirname $(realpath $0)` _here=`dirname $(realpath $0)`
XMLPARSE="${_here}/helpers/tf-xml-filelist.py" XMLPARSE="${_here}/helpers/tf-xml-filelist.py"
INDEXGEN="${_here}/helpers/tf-gen-index.py"
TF_UPSTREAM_BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://storage.googleapis.com/tensorflow"} TF_UPSTREAM_BASE_URL=${TUNASYNC_UPSTREAM_URL:-"https://storage.googleapis.com/tensorflow"}
BASE_PATH="${TUNASYNC_WORKING_DIR}" BASE_PATH="${TUNASYNC_WORKING_DIR}"
@ -41,14 +42,7 @@ wget -O - "${TF_UPSTREAM_BASE_URL}/" | ${XMLPARSE} | while read -a tokens; do
fi fi
done done
find ${BASE_PATH} -type f -name '*.whl' -printf '%P\n' | \
${INDEXGEN} > "${BASE_PATH}/releases.json"
exit $failed exit $failed

178
test/tf.filelist Normal file
View File

@ -0,0 +1,178 @@
linux/gpu/tensorflow-0.10.0-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow-0.10.0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow_gpu-0.12.0rc0-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow_gpu-0.12.0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0rc0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow-0.10.0rc0-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow-0.5.0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.9.0rc0-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.8.0rc0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow-0.6.0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow_gpu-0.12.0rc1-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.7.1-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow-0.8.0rc0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.9.0rc0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.9.0rc0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow-0.6.0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.10.0rc0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow_gpu-0.12.0rc0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.10.0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0rc0-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow_gpu-0.12.0-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow_gpu-0.12.0rc0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow_gpu-0.12.0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.6.0-cp34-none-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow-0.7.1-cp34-none-linux_x86_64.whl
linux/gpu/tensorflow_gpu-0.12.0rc1-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow-0.7.0-py2-none-linux_x86_64.whl
linux/gpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow_gpu-0.12.0rc1-cp35-cp35m-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.11.0rc0-cp27-none-linux_x86_64.whl
linux/gpu/tensorflow-0.7.0-py3-none-linux_x86_64.whl
linux/gpu/tensorflow-0.10.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.8.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.6.0-cp34-none-linux_x86_64.whl
linux/cpu/tensorflow-0.12.0-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.7.0-py2-none-linux_x86_64.whl
linux/cpu/protobuf-3.0.0-cp35-cp35m-linux_x86_64.whl
linux/cpu/tensorflow-0.7.1-cp34-none-linux_x86_64.whl
linux/cpu/tensorflow-0.8.0-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.9.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/protobuf-3.0.0b2.post2-cp34-none-linux_x86_64.whl
linux/cpu/tensorflow-0.9.0-cp35-cp35m-linux_x86_64.whl
linux/cpu/tensorflow-0.10.0rc0-cp35-cp35m-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0rc0-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0rc2-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.12.0rc1-cp35-cp35m-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0rc2-cp35-cp35m-linux_x86_64.whl
linux/cpu/tensorflow-0.12.0rc1-cp34-cp34m-linux_x86_64.whl
linux/cpu/protobuf-3.0.0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0rc0-cp35-cp35m-linux_x86_64.whl
linux/cpu/tensorflow-0.12.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.7.0-py3-none-linux_x86_64.whl
linux/cpu/cloudml/tensorflow-0.11.0-cp27-none-linux_x86_64.whl
linux/cpu/cloudml/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl
linux/cpu/cloudml/tensorflow-0.12.0rc1-cp27-none-linux_x86_64.whl
linux/cpu/cloudml/tensorflow-0.12.0-cp27-none-linux_x86_64.whl
linux/cpu/cloudml/tensorflow-0.11.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/cloudml/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl
linux/cpu/cloudml/tensorflow-0.12.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.10.0rc0-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl
linux/cpu/protobuf-3.0.0-cp3-none-linux_x86_64.whl
linux/cpu/protobuf-3.0.0-cp34-cp34m-linux_x86_64.whl
linux/cpu/protobuf-3.1.0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.10.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/protobuf-3.0.0b3-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.12.0-cp35-cp35m-linux_x86_64.whl
linux/cpu/tensorflow-0.12.0rc1-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.10.0-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.5.0-cp27-none-linux_x86_64.whl
linux/cpu/protobuf-3.0.0-cp27-cp27mu-linux_x86_64.whl
linux/cpu/tensorflow-0.12.0rc0-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0rc1-cp35-cp35m-linux_x86_64.whl
linux/cpu/tensorflow-0.10.0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.9.0rc0-cp35-cp35m-linux_x86_64.whl
linux/cpu/tensorflow-0.6.0-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0-cp35-cp35m-linux_x86_64.whl
linux/cpu/protobuf-3.0.0b3-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.8.0rc0-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0-cp34-cp34m-linux_x86_64.whl
linux/cpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.12.0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.9.0rc0-cp34-cp34m-linux_x86_64.whl
linux/cpu/protobuf-3.0.0b2.post2-cp27-none-linux_x86_64.whl
linux/cpu/protobuf-3.1.0-cp35-none-linux_x86_64.whl
linux/cpu/tensorflow-0.8.0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.7.1-cp27-none-linux_x86_64.whl
linux/cpu/protobuf-3.1.0-cp27-cp27mu-linux_x86_64.whl
linux/cpu/tensorflow-0.10.0-cp35-cp35m-linux_x86_64.whl
linux/cpu/tensorflow-0.6.0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.11.0rc1-cp34-cp34m-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.12.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.11.0-cp27-none-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.11.0rc1-cp27-none-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.10.0-cp27-none-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.11.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.9.0-cp27-none-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.12.0rc1-cp27-none-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.10.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.11.0rc2-cp27-none-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.12.0-cp27-none-linux_x86_64.whl
linux/cpu/debian/jessie/tensorflow-0.9.0rc0-cp27-none-linux_x86_64.whl
linux/cpu/tensorflow-0.12.0rc0-cp35-cp35m-linux_x86_64.whl
mac/tensorflow-0.6.0-py2-none-any.whl
mac/tensorflow-0.6.0-py3-none-any.whl
mac/tensorflow-0.8.0rc0-py2-none-any.whl
mac/tensorflow-0.8.0rc0-py3-none-any.whl
mac/tensorflow-0.7.1-cp35-none-any.whl
mac/tensorflow-0.8.0-py3-none-any.whl
mac/tensorflow-0.8.0-py2-none-any.whl
mac/tensorflow-0.9.0rc0-py2-none-any.whl
mac/cpu/tensorflow-0.12.0-py2-none-any.whl
mac/cpu/tensorflow-0.12.0-py3-none-any.whl
mac/cpu/tensorflow-0.10.0rc0-py3-none-any.whl
mac/cpu/tensorflow-0.10.0rc0-py2-none-any.whl
mac/cpu/tensorflow-0.10.0-py2-none-any.whl
mac/cpu/tensorflow-0.11.0rc2-py2-none-any.whl
mac/cpu/tensorflow-0.11.0rc2-py3-none-any.whl
mac/cpu/tensorflow-0.10.0-py3-none-any.whl
mac/cpu/tensorflow-0.11.0rc0-py3-none-any.whl
mac/cpu/tensorflow-0.11.0rc0-py2-none-any.whl
mac/cpu/protobuf-3.0.0-cp3-cp3m-macosx_10_11_x86_64.whl
mac/cpu/tensorflow-0.11.0rc1-py3-none-any.whl
mac/cpu/tensorflow-0.11.0rc1-py2-none-any.whl
mac/cpu/tensorflow-0.12.0rc0-py2-none-any.whl
mac/cpu/protobuf-3.0.0-cp35-cp35m-macosx_10_11_x86_64.whl
mac/cpu/tensorflow-0.11.0-py3-none-any.whl
mac/cpu/tensorflow-0.11.0-py2-none-any.whl
mac/cpu/tensorflow-0.12.0rc0-py3-none-any.whl
mac/cpu/tensorflow-0.12.0rc1-py2-none-any.whl
mac/cpu/tensorflow-0.12.0rc1-py3-none-any.whl
mac/cpu/protobuf-3.0.0-cp27-cp27m-macosx_10_11_x86_64.whl
mac/cpu/protobuf-3.1.0-cp27-none-macosx_10_11_x86_64.whl
mac/cpu/protobuf-3.1.0-cp35-none-macosx_10_11_x86_64.whl
mac/protobuf-3.0.0b2.post2-cp35-none-any.whl
mac/tensorflow-0.9.0rc0-py3-none-any.whl
mac/gpu/tensorflow_gpu-0.12.0rc1-py3-none-any.whl
mac/gpu/tensorflow_gpu-0.12.0rc1-py2-none-any.whl
mac/gpu/tensorflow_gpu-0.12.0rc0-py3-none-any.whl
mac/gpu/tensorflow-0.10.0rc0-py3-none-any.whl
mac/gpu/tensorflow_gpu-0.12.0rc0-py2-none-any.whl
mac/gpu/tensorflow-0.10.0rc0-py2-none-any.whl
mac/gpu/tensorflow-0.11.0rc1-py3-none-any.whl
mac/gpu/tensorflow-0.10.0-py3-none-any.whl
mac/gpu/tensorflow-0.10.0-py2-none-any.whl
mac/gpu/tensorflow-0.11.0rc1-py2-none-any.whl
mac/gpu/tensorflow-0.11.0rc0-py3-none-any.whl
mac/gpu/tensorflow-0.11.0rc0-py2-none-any.whl
mac/gpu/tensorflow-0.11.0rc2-py2-none-any.whl
mac/gpu/tensorflow-0.11.0rc2-py3-none-any.whl
mac/gpu/tensorflow-0.11.0-py2-none-any.whl
mac/gpu/tensorflow-0.11.0-py3-none-any.whl
mac/gpu/tensorflow_gpu-0.12.0-py3-none-any.whl
mac/gpu/tensorflow_gpu-0.12.0-py2-none-any.whl
mac/protobuf-3.0.0b2.post2-cp27-none-any.whl
mac/tensorflow-0.7.1-cp27-none-any.whl
mac/tensorflow-0.5.0-py2-none-any.whl
mac/tensorflow-0.9.0-py2-none-any.whl
mac/tensorflow-0.9.0-py3-none-any.whl
mac/tensorflow-0.7.0-py3-none-any.whl
mac/tensorflow-0.7.0-py2-none-any.whl