From 5621913bf1be278d003423178d9e200a2b742c3f Mon Sep 17 00:00:00 2001 From: Nanthini Date: Tue, 10 Aug 2021 12:31:29 -0500 Subject: [PATCH 1/3] Use RAPIDS docker image instead of cuda --- Dockerfile | 133 +++++++---------------------------------------------- 1 file changed, 17 insertions(+), 116 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb41d32..2ce5b04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,103 +1,21 @@ -ARG CUDA_VERSION -FROM nvidia/cuda:$CUDA_VERSION-devel-ubuntu18.04 -SHELL ["/bin/bash", "-c"] -# Install conda (and use python 3.7) -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - doxygen \ - git \ - graphviz \ - libcurl4-openssl-dev \ - libboost-all-dev \ - make \ - tar \ - unzip \ - wget \ - zlib1g-dev && \ - rm -rf /var/lib/apt/* +ARG RAPIDS_VER="21.06" +ARG CUDA_VER="11.0" +ARG IMG_TYPE="base" +ARG LINUX_VER="ubuntu18.04" +ARG PYTHON_VER="3.8" +FROM rapidsai/rapidsai-core:${RAPIDS_VER}-cuda${CUDA_VER}-${IMG_TYPE}-${LINUX_VER}-py${PYTHON_VER} -RUN curl -o /opt/miniconda.sh \ - https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - chmod +x /opt/miniconda.sh && \ - /opt/miniconda.sh -b -p /opt/conda && \ - /opt/conda/bin/conda update -n base conda && \ - rm /opt/miniconda.sh -ENV PATH /opt/conda/bin:$PATH -RUN conda install -c conda-forge -c rapidsai -c nvidia -c defaults \ - bokeh \ - cmake>=3.14 \ - h5py \ - ipython \ - ipywidgets \ - jupyter \ - kaggle \ - matplotlib \ - nose \ - numpy \ - pandas \ - Pillow \ - pydot \ - pylint\ - psutil\ - scikit-learn \ - scipy \ - six \ - dask \ - distributed \ - tqdm \ - cudf=0.18.0 \ - dask-cuda \ - rmm \ - librmm \ - rapids-xgboost \ - cuml=0.18 && \ - conda clean -ya +ENV PYTHONUNBUFFERED=True -# lightgbm -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - bzip2 \ - ca-certificates \ - curl \ - git \ - libblas-dev \ - libboost-dev \ - libboost-filesystem-dev \ - libboost-system-dev \ - libbz2-dev \ - libc6 \ - libglib2.0-0 \ - liblapack-dev \ - libsm6 \ - libxext6 \ - libxrender1 \ - make \ - tar \ - unzip \ - wget && \ - rm -rf /var/lib/apt/* -RUN mkdir -p /etc/OpenCL/vendors && \ - echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd -ENV OPENCL_LIBRARIES /usr/local/cuda/lib64 -ENV OPENCL_INCLUDE_DIR /usr/local/cuda/include -RUN git config --global http.sslVerify false && \ - git clone --recursive https://github.com/Microsoft/LightGBM /opt/LightGBM && \ - cd /opt/LightGBM && \ - mkdir build && \ - cd build && \ - cmake .. \ - -DUSE_GPU=1 \ - -DOpenCL_LIBRARY=$OPENCL_LIBRARIES/libOpenCL.so \ - -DOpenCL_INCLUDE_DIR=$OPENCL_INCLUDE_DIR && \ - make OPENCL_HEADERS="/usr/local/cuda/targets/x86_64-linux/include" \ - LIBOPENCL="/usr/local/cuda/targets/x86_64-linux/lib" -j4 && \ - cd ../python-package && \ - python setup.py install --precompile +RUN apt update -y \ + && apt install -y --no-install-recommends build-essential \ + && apt install -y cmake \ + && apt autoremove -y \ + && apt clean -y \ + && rm -rf /var/lib/apt/lists/* + +RUN . /opt/conda/etc/profile.d/conda.sh \ + && conda activate rapids # catboost RUN if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then git config --global http.sslVerify false && \ @@ -108,25 +26,8 @@ RUN if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then git config -r \ -o ../../.. \ -DUSE_ARCADIA_PYTHON=no \ - -DUSE_SYSTEM_PYTHON=3.7\ + -DUSE_SYSTEM_PYTHON={PYTHON_VER}\ -DPYTHON_CONFIG=python3-config \ -DCUDA_ROOT=$(dirname $(dirname $(which nvcc))); \ fi ENV if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then PYTHONPATH=$PYTHONPATH:/opt/catboost/catboost/python-package; fi - - - -# xgboost -RUN git config --global http.sslVerify false && \ - git clone --recursive https://github.com/dmlc/xgboost /opt/xgboost && \ - cd /opt/xgboost && \ - mkdir build && \ - cd build && \ - RMM_ROOT=/opt/conda cmake .. \ - -DUSE_CUDA=ON \ - -DUSE_NCCL=ON \ - -DPLUGIN_RMM=ON && \ - make -j4 && \ - cd ../python-package && \ - pip uninstall -y xgboost && \ - python setup.py install From 1c8cf06e45dae1487452c592f53734ab5ea4e402 Mon Sep 17 00:00:00 2001 From: Nanthini Date: Thu, 12 Aug 2021 15:12:49 -0500 Subject: [PATCH 2/3] NOT working dockerfile --- Dockerfile | 55 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2ce5b04..cf24b04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,44 @@ ARG RAPIDS_VER="21.06" ARG CUDA_VER="11.0" -ARG IMG_TYPE="base" +ARG IMG_TYPE="devel" ARG LINUX_VER="ubuntu18.04" ARG PYTHON_VER="3.8" -FROM rapidsai/rapidsai-core:${RAPIDS_VER}-cuda${CUDA_VER}-${IMG_TYPE}-${LINUX_VER}-py${PYTHON_VER} -ENV PYTHONUNBUFFERED=True +FROM rapidsai/rapidsai-dev:${RAPIDS_VER}-cuda${CUDA_VER}-${IMG_TYPE}-${LINUX_VER}-py${PYTHON_VER} -RUN apt update -y \ - && apt install -y --no-install-recommends build-essential \ - && apt install -y cmake \ - && apt autoremove -y \ - && apt clean -y \ - && rm -rf /var/lib/apt/lists/* +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential RUN . /opt/conda/etc/profile.d/conda.sh \ && conda activate rapids # catboost -RUN if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then git config --global http.sslVerify false && \ - git clone --recursive "https://github.com/catboost/catboost" /opt/catboost && \ - cd /opt/catboost && \ - cd catboost/python-package/catboost && \ - ../../../ya make \ - -r \ - -o ../../.. \ - -DUSE_ARCADIA_PYTHON=no \ - -DUSE_SYSTEM_PYTHON={PYTHON_VER}\ - -DPYTHON_CONFIG=python3-config \ - -DCUDA_ROOT=$(dirname $(dirname $(which nvcc))); \ - fi -ENV if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then PYTHONPATH=$PYTHONPATH:/opt/catboost/catboost/python-package; fi +# RUN if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then git config --global http.sslVerify false && \ +# git clone --recursive "https://github.com/catboost/catboost" /opt/catboost && \ +# cd /opt/catboost && \ +# cd catboost/python-package/catboost && \ +# ../../../ya make \ +# -r \ +# -o ../../.. \ +# -DUSE_ARCADIA_PYTHON=no \ +# -DUSE_SYSTEM_PYTHON={PYTHON_VER}\ +# -DPYTHON_CONFIG=python3-config \ +# -DCUDA_ROOT=$(dirname $(dirname $(which nvcc))); \ +# fi +# ENV if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then PYTHONPATH=$PYTHONPATH:/opt/catboost/catboost/python-package; fi + +# xgboost +RUN git config --global http.sslVerify false && \ + git clone --recursive https://github.com/dmlc/xgboost /opt/xgboost && \ + cd /opt/xgboost && \ + mkdir build && \ + cd build && \ + RMM_ROOT=/opt/conda cmake .. \ + -DUSE_CUDA=ON \ + -DUSE_NCCL=ON \ + -DPLUGIN_RMM=ON && \ + make -j4 && \ + cd ../python-package && \ + pip uninstall -y xgboost && \ + python setup.py install From 837cdeff449c6dd6a6851f9580ccaef1b6755d59 Mon Sep 17 00:00:00 2001 From: Nanthini Date: Fri, 13 Aug 2021 12:03:21 -0500 Subject: [PATCH 3/3] UPDATE xgboost install --- Dockerfile | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf24b04..a40924f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,19 +14,19 @@ RUN . /opt/conda/etc/profile.d/conda.sh \ && conda activate rapids # catboost -# RUN if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then git config --global http.sslVerify false && \ -# git clone --recursive "https://github.com/catboost/catboost" /opt/catboost && \ -# cd /opt/catboost && \ -# cd catboost/python-package/catboost && \ -# ../../../ya make \ -# -r \ -# -o ../../.. \ -# -DUSE_ARCADIA_PYTHON=no \ -# -DUSE_SYSTEM_PYTHON={PYTHON_VER}\ -# -DPYTHON_CONFIG=python3-config \ -# -DCUDA_ROOT=$(dirname $(dirname $(which nvcc))); \ -# fi -# ENV if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then PYTHONPATH=$PYTHONPATH:/opt/catboost/catboost/python-package; fi +RUN if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then git config --global http.sslVerify false && \ + git clone --recursive "https://github.com/catboost/catboost" /opt/catboost && \ + cd /opt/catboost && \ + cd catboost/python-package/catboost && \ + ../../../ya make \ + -r \ + -o ../../.. \ + -DUSE_ARCADIA_PYTHON=no \ + -DUSE_SYSTEM_PYTHON={PYTHON_VER}\ + -DPYTHON_CONFIG=python3-config \ + -DCUDA_ROOT=$(dirname $(dirname $(which nvcc))); \ + fi +ENV if [ "`echo $CUDA_VERSION | sed -e 's/[.].*//'`" -lt "11" ]; then PYTHONPATH=$PYTHONPATH:/opt/catboost/catboost/python-package; fi # xgboost RUN git config --global http.sslVerify false && \ @@ -34,7 +34,7 @@ RUN git config --global http.sslVerify false && \ cd /opt/xgboost && \ mkdir build && \ cd build && \ - RMM_ROOT=/opt/conda cmake .. \ + RMM_ROOT=/opt/conda/envs/rapids cmake .. \ -DUSE_CUDA=ON \ -DUSE_NCCL=ON \ -DPLUGIN_RMM=ON && \