From b14eac015bc3fe813d8d37cdf7eff54a5b15e526 Mon Sep 17 00:00:00 2001 From: Johnny Chen Date: Thu, 28 May 2020 09:17:30 +0800 Subject: [PATCH 1/2] use StorageServer#v0.1.0.beta and remove jill dependency --- dockerfiles/julia/Dockerfile | 24 +++++++++--------------- julia.sh | 4 ++++ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/dockerfiles/julia/Dockerfile b/dockerfiles/julia/Dockerfile index 8450d60..98471cb 100644 --- a/dockerfiles/julia/Dockerfile +++ b/dockerfiles/julia/Dockerfile @@ -2,30 +2,24 @@ # * https://github.com/tuna/tunasync-scripts/pull/71 # * https://github.com/tuna/issues/issues/837 -# jill.py requires at least Python 3.6 -FROM python:3.6 +# StorageServer.jl is used to set up a *static* storage server for julia packages, it requires at least Julia 1.4 +# The details of the storage protocol can be found in https://github.com/JuliaLang/Pkg.jl/issues/1377 +FROM julia:1.4 LABEL description="A community maintained docker script to set up julia mirror easily." LABEL maintainer="Johnny Chen " +RUN apt-get update && \ + apt-get install -y \ + git && \ + rm -rf /var/lib/apt/lists/* + ENV JULIA_DEPOT_PATH="/tmp/julia" ENV JULIA_STATIC_DIR="/julia/static" ENV JULIA_CLONES_DIR="/julia/clones" -# jill.py provides two functionalities: -# 1) mirror julia binary releases -# 2) install julia in one line -# FYI: the jill API is quite stable so the version doesn't matter much (it could be okay to set >=0.6.9) -RUN pip install --upgrade pip && \ - pip install jill==0.6.9 && \ - pip cache purge - -# StorageServer.jl is used to set up a *static* storage server for julia packages, it requires at least Julia 1.4 -# The details of the storage protocol can be found in https://github.com/JuliaLang/Pkg.jl/issues/1377 -RUN jill install 1.4 --confirm - # StorageServer.jl is an experimental toolkit and it won't be registered in General # The API is likely to be changed in the future, so we fix the version here for stability consideration -RUN julia -e 'using Pkg; pkg"add https://github.com/johnnychen94/StorageServer.jl#v0.1.0-beta.1"' +RUN julia -e 'using Pkg; pkg"add https://github.com/johnnychen94/StorageServer.jl#v0.1.0-beta"' RUN chown -R 2000 /tmp/julia/ diff --git a/julia.sh b/julia.sh index 1dc0455..b4590b1 100755 --- a/julia.sh +++ b/julia.sh @@ -6,5 +6,9 @@ cd "${TUNASYNC_WORKING_DIR}" export JULIA_STATIC_DIR="$PWD/static" export JULIA_CLONES_DIR="$PWD/clones" + +# update and mirror the General registry +git -C registries/General fetch --all +git -C registries/General reset --hard origin/master exec julia -e "using StorageServer; mirror_tarball(\"registries/General\", [\"$BASE_URL\"])" From 306960e39b0b4643fbbdc290791c57eb20a9df7d Mon Sep 17 00:00:00 2001 From: Johnny Chen Date: Fri, 29 May 2020 12:58:24 +0800 Subject: [PATCH 2/2] skip packages that unexpectedly take a long time --- dockerfiles/julia/Dockerfile | 2 +- julia.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dockerfiles/julia/Dockerfile b/dockerfiles/julia/Dockerfile index 98471cb..9229fdd 100644 --- a/dockerfiles/julia/Dockerfile +++ b/dockerfiles/julia/Dockerfile @@ -19,7 +19,7 @@ ENV JULIA_CLONES_DIR="/julia/clones" # StorageServer.jl is an experimental toolkit and it won't be registered in General # The API is likely to be changed in the future, so we fix the version here for stability consideration -RUN julia -e 'using Pkg; pkg"add https://github.com/johnnychen94/StorageServer.jl#v0.1.0-beta"' +RUN julia -e 'using Pkg; pkg"add https://github.com/johnnychen94/StorageServer.jl#v0.1.0-rc1"' RUN chown -R 2000 /tmp/julia/ diff --git a/julia.sh b/julia.sh index b4590b1..4a56062 100755 --- a/julia.sh +++ b/julia.sh @@ -7,8 +7,12 @@ cd "${TUNASYNC_WORKING_DIR}" export JULIA_STATIC_DIR="$PWD/static" export JULIA_CLONES_DIR="$PWD/clones" +# timeout (seconds) for individual package instead of the whole mirror process +# initialization should use a larger timeout, e.g., 7200 +PKG_TIMEOUT=600 + # update and mirror the General registry git -C registries/General fetch --all git -C registries/General reset --hard origin/master -exec julia -e "using StorageServer; mirror_tarball(\"registries/General\", [\"$BASE_URL\"])" +exec julia -e "using StorageServer; mirror_tarball(\"registries/General\", [\"$BASE_URL\"]; timeout=$PKG_TIMEOUT)"