From de2544b0bf0959da731df8b8d1661a2a2f364b80 Mon Sep 17 00:00:00 2001 From: Mehdi Othmani-Guibourg Date: Sun, 19 Jul 2020 20:01:37 +0200 Subject: [PATCH 1/2] Docker image and make commands to run Pegasus with TensorFlow 1.15 and CUDA 10 --- docker/Dockerfile | 36 ++++++++++++++++++++++++++++++++ docker/Makefile | 23 +++++++++++++++++++++ docker/README.md | 41 +++++++++++++++++++++++++++++++++++++ docker/bash.bashrc | 39 +++++++++++++++++++++++++++++++++++ docker/docker_entrypoint.sh | 8 ++++++++ 5 files changed, 147 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/Makefile create mode 100644 docker/README.md create mode 100755 docker/bash.bashrc create mode 100644 docker/docker_entrypoint.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..e5fd9f65 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,36 @@ +FROM tensorflow/tensorflow:1.15.2-gpu + +ARG USER=pegasus +ARG USER_UID=1000 + +RUN apt-get update && apt-get install -y \ + git \ + man \ + nano + +RUN mkdir /src \ + && git clone https://github.com/google-research/pegasus /src/ + +RUN groupadd ${USER} && useradd -m -u ${USER_UID} -g ${USER} -s /bin/bash ${USER} \ + && chown -R ${USER} /src + +USER ${USER} + +ENV PATH="/home/${USER}/.local/bin:$PATH" + +RUN pip3 install --user --upgrade pip \ + && pip3 install --user six==1.14.0 protobuf==3.12.0 cloudpickle==1.2.0 gsutil jupyter \ + && pip3 install --user -r /src/requirements.txt + +RUN mkdir /src/ckpt/ + +COPY docker_entrypoint.sh /src/ + +COPY bash.bashrc /etc/ + +WORKDIR /src + +ENV PYTHONPATH="/src/:$PYTHONPATH" + +ENTRYPOINT ["sh", "/src/docker_entrypoint.sh"] +CMD ["sh"] diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 00000000..b01eac1c --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,23 @@ +help: + @cat Makefile + +IMAGE=google/pegasus +PROJECT=pegasus +DOCKER_FILE=Dockerfile +SRC?=$(shell dirname `pwd`) +DATA?="${HOME}/Data" + +build: + docker build -t $(IMAGE) -f $(DOCKER_FILE) . + +bash: build + docker run --init -it --gpus all --hostname $(PROJECT)-run -v $(SRC):/src/workspace -v $(DATA):/data --mount type=volume,source=$(PROJECT)-data,target=/src/ckpt/ --name $(PROJECT)-run $(IMAGE) bash + +ipython: build + docker run --init -it --gpus all --hostname $(PROJECT)-run -v $(SRC):/src/workspace -v $(DATA):/data --mount type=volume,source=$(PROJECT)-data,target=/src/ckpt/ --name $(PROJECT)-run $(IMAGE) ipython + +notebook: build + docker run --init -it --gpus all --hostname $(PROJECT)-run -v $(SRC):/src/workspace -v $(DATA):/data --mount type=volume,source=$(PROJECT)-data,target=/src/ckpt/ --net=host --name $(PROJECT)-run $(IMAGE) jupyter notebook --port=8888 --ip=0.0.0.0 + +test: build + docker run --init -it --gpus all --hostname $(PROJECT)-run -v $(SRC):/src/workspace -v $(DATA):/data --mount type=volume,source=$(PROJECT)-data,target=/src/ckpt/ --name $(PROJECT) $(IMAGE) python3 pegasus/bin/train.py --params=aeslc_transformer --param_overrides=vocab_filename=ckpt/pegasus_ckpt/c4.unigram.newline.10pct.96000.model --train_init_checkpoint=ckpt/pegasus_ckpt/model.ckpt-1500000 --model_dir=ckpt/pegasus_ckpt/aeslc diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..2955c540 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,41 @@ +# Using PEGASUS via Docker + +This directory contains the `Dockerfile` to make it easy to get up and running with +PEGASUS via [Docker](http://www.docker.com/). + +## Installing Docker + +General installation instructions are +[on the Docker site](https://docs.docker.com/installation/). + +## Installing GPU support + +For GPU support install NVIDIA drivers (ideally the latest) and the [NVIDIA Container Toolkit](https://github.com/NVIDIA/nvidia-docker). + +## Running the container + +`Makefile` is used to simplify docker commands within `make` commands. + +Build the container and start a bash + + $ make bash + +Build the container and start an iPython shell + + $ make ipython + +Build the container and start a Jupyter Notebook + + $ make notebook + +Build the container and start the finetuning test described in the repository's README: + + $ make test + +Mount a volume for external data sets + + $ make DATA=~/mydata + +Prints all make tasks + + $ make help diff --git a/docker/bash.bashrc b/docker/bash.bashrc new file mode 100755 index 00000000..672e0f95 --- /dev/null +++ b/docker/bash.bashrc @@ -0,0 +1,39 @@ +# +# /etc/bash.bashrc +# + +export PS1="\[\e[31m\]pegasus-docker\[\e[m\] \[\e[33m\]\w\[\e[m\] > " +export TERM=xterm-256color +alias grep="grep --color=auto" +alias ls="ls --color=auto" + +echo -e "\e[1;31m" +cat<

__ \_\___ \| | /\___ \ + |____| \___ >___ (____ /____ >____//____ > + \/_____/ \/ \/ \/ + +P +echo -e "\e[0;33m" + +if [[ $EUID -eq 0 ]]; then + cat < Date: Sat, 1 Aug 2020 18:54:35 +0200 Subject: [PATCH 2/2] Improvement of Dockerfile: the GPUs are now detected using Anaconda --- docker/Dockerfile | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e5fd9f65..da38b460 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,23 +4,44 @@ ARG USER=pegasus ARG USER_UID=1000 RUN apt-get update && apt-get install -y \ + wget \ git \ man \ nano +ENV CONDA_PATH /opt/conda +ENV PATH ${CONDA_PATH}/bin:${PATH} + +RUN wget --quiet --no-check-certificate https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.3-Linux-x86_64.sh \ + && echo "bb2e3cedd2e78a8bb6872ab3ab5b1266a90f8c7004a22d8dc2ea5effeb6a439a *Miniconda3-py37_4.8.3-Linux-x86_64.sh" | sha256sum -c - \ + && /bin/bash Miniconda3-py37_4.8.3-Linux-x86_64.sh -f -b -p ${CONDA_PATH} \ + && rm Miniconda3-py37_4.8.3-Linux-x86_64.sh \ + && echo export PATH=${CONDA_PATH}/bin:'${PATH}' > /etc/profile.d/conda.sh + RUN mkdir /src \ && git clone https://github.com/google-research/pegasus /src/ RUN groupadd ${USER} && useradd -m -u ${USER_UID} -g ${USER} -s /bin/bash ${USER} \ - && chown -R ${USER} /src + && chown -R ${USER_UID} /src \ + && chown -R ${USER_UID} ${CONDA_PATH} USER ${USER} -ENV PATH="/home/${USER}/.local/bin:$PATH" - -RUN pip3 install --user --upgrade pip \ - && pip3 install --user six==1.14.0 protobuf==3.12.0 cloudpickle==1.2.0 gsutil jupyter \ - && pip3 install --user -r /src/requirements.txt +# ENV PATH="/home/${USER}/.local/bin:$PATH" + +ARG python_version=3.6 + +RUN conda config --append channels conda-forge +RUN conda install -y \ + python=${python_version} \ + notebook \ + && pip install --upgrade pip \ + && pip install \ + six==1.14.0 \ + protobuf==3.12.0 \ + cloudpickle==1.2.0 \ + gsutil \ + && pip install -r /src/requirements.txt RUN mkdir /src/ckpt/