From c5fd8f27723cc835227573d3e4d3248a8e84b317 Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam (macpro2)" Date: Fri, 2 Feb 2018 20:43:41 -0800 Subject: [PATCH 1/3] updating docker to support bigdl v0.4 --- elephantscale/docker/Dockerfile | 20 +++++++++++--------- elephantscale/run-bigdl-docker.sh | 8 ++------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/elephantscale/docker/Dockerfile b/elephantscale/docker/Dockerfile index 0e9a549..f375ab8 100644 --- a/elephantscale/docker/Dockerfile +++ b/elephantscale/docker/Dockerfile @@ -1,7 +1,7 @@ ## pick a specific version, to ensure predictability -FROM jupyter/pyspark-notebook@sha256:64420e4c348ab48fb806f42332109cbc205ae74cda67240c3e0974c5f7e6e969 +FROM jupyter/pyspark-notebook@sha256:7afe62876ec622b56f707d3adcafae4fd8b29e3b78863211091724b8a902837f ## or latest -#FROM jupyter/pyspark-notebook@latest +#FROM jupyter/pyspark-notebook:latest MAINTAINER Elephant Scale @@ -13,10 +13,9 @@ ARG SBT_VERSION=1.0.2 ARG INSTALL_DIR=/usr/local ENV BIGDL_HOME ${INSTALL_DIR}/BigDL -#ARG BIGDL_URL=https://s3.amazonaws.com/elephantscale-public/BigDL/BigDL.zip ## Download BigDL from release page -## https://bigdl-project.github.io/0.3.0/#release-download/ -ARG BIGDL_URL=https://repo1.maven.org/maven2/com/intel/analytics/bigdl/dist-spark-${SPARK_VERSION}-scala-2.11.8-linux64/0.3.0/dist-spark-${SPARK_VERSION}-scala-2.11.8-linux64-0.3.0-dist.zip +## https://repo1.maven.org/maven2/com/intel/analytics/bigdl/ +ARG BIGDL_URL=https://repo1.maven.org/maven2/com/intel/analytics/bigdl/dist-spark-2.2.0-scala-2.11.8-all/0.4.0/dist-spark-2.2.0-scala-2.11.8-all-0.4.0-dist.zip ## --- end CONFIG USER root @@ -30,7 +29,7 @@ RUN apt-get install -yq --no-install-recommends \ atop \ curl \ less \ - openjdk-8-jdk-headless \ + openjdk-8-jdk \ rsync \ unzip \ wget \ @@ -76,6 +75,7 @@ RUN apt-get clean && \ rm -rf /var/lib/apt/lists/* ## ----- install BigDL +RUN echo "BIGDL_URL=$BIGDL_URL" RUN \ mkdir -p ${BIGDL_HOME} && \ cd ${BIGDL_HOME} && \ @@ -96,15 +96,17 @@ RUN mkdir /work USER $NB_USER ## update conda -# RUN conda update --all +RUN conda update --all ## install NLTK RUN conda install -y nltk ## python 3.5 env -RUN conda create -y -n py35 python=3.5 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk tensorflow opencv pillow +RUN conda create -y -n py35 python=3.5 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk opencv pillow tensorflow +#tensorflow-tensorboard ## python 2.7 env -RUN conda create -y -n py27 python=2.7 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk tensorflow opencv pillow +RUN conda create -y -n py27 python=2.7 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk opencv pillow tensorflow +# tensorflow-tensorboard # list envs RUN conda info -e diff --git a/elephantscale/run-bigdl-docker.sh b/elephantscale/run-bigdl-docker.sh index 6d01f68..cc1acc9 100755 --- a/elephantscale/run-bigdl-docker.sh +++ b/elephantscale/run-bigdl-docker.sh @@ -3,7 +3,7 @@ ## Usage # ./run-bigdl-docker.sh [optional command] # -# ./run-bigdl-docker.sh xxx/yyy +# ./run-bigdl-docker.sh intel/bigdl # # or during developing, give a local docker image id # ./run-bigdl-docker.sh abcd1234 @@ -19,16 +19,12 @@ fi image_id="$1" cmd="$2" -name="bigdl" - -## remove any previously running containers -docker rm -f "$name" # mount the current directory at /work this="${BASH_SOURCE-$0}" mydir=$(cd -P -- "$(dirname -- "$this")" && pwd -P) -docker run -it --name "$name" \ +docker run -it \ -p 8888:8888 \ -p 6006:6006 \ -v"$mydir:/work" \ From e7d489c91b2138005e1ed5ef450849f2f02c35a6 Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam (macpro2)" Date: Sat, 3 Feb 2018 18:25:05 -0800 Subject: [PATCH 2/3] minor fixes to Dockerfile --- elephantscale/docker/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/elephantscale/docker/Dockerfile b/elephantscale/docker/Dockerfile index f375ab8..a7e32ef 100644 --- a/elephantscale/docker/Dockerfile +++ b/elephantscale/docker/Dockerfile @@ -1,5 +1,5 @@ ## pick a specific version, to ensure predictability -FROM jupyter/pyspark-notebook@sha256:7afe62876ec622b56f707d3adcafae4fd8b29e3b78863211091724b8a902837f +FROM jupyter/pyspark-notebook@sha256:f5f1b391d3f6e94ae864690db1c5e0e9ffecf2f4a0a00f09c01c3e3a9230638e ## or latest #FROM jupyter/pyspark-notebook:latest @@ -96,17 +96,15 @@ RUN mkdir /work USER $NB_USER ## update conda -RUN conda update --all +#RUN conda update --all ## install NLTK RUN conda install -y nltk ## python 3.5 env RUN conda create -y -n py35 python=3.5 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk opencv pillow tensorflow -#tensorflow-tensorboard ## python 2.7 env RUN conda create -y -n py27 python=2.7 numpy scipy pandas scikit-learn matplotlib seaborn jupyter nltk opencv pillow tensorflow -# tensorflow-tensorboard # list envs RUN conda info -e From ee1b44a467d96b9b3fb07b5b256ef6c8197d8d5d Mon Sep 17 00:00:00 2001 From: "Sujee Maniyam (macpro2)" Date: Sat, 3 Feb 2018 18:34:29 -0800 Subject: [PATCH 3/3] updating docker hub name for run script --- elephantscale/run-bigdl-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elephantscale/run-bigdl-docker.sh b/elephantscale/run-bigdl-docker.sh index cc1acc9..28f39e3 100755 --- a/elephantscale/run-bigdl-docker.sh +++ b/elephantscale/run-bigdl-docker.sh @@ -3,7 +3,7 @@ ## Usage # ./run-bigdl-docker.sh [optional command] # -# ./run-bigdl-docker.sh intel/bigdl +# ./run-bigdl-docker.sh elephantscale/bigdl-sandbox # # or during developing, give a local docker image id # ./run-bigdl-docker.sh abcd1234