diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2ebb488 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +.github +README.md +image-variants.json +scripts +tests diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 4739e02..80124bb 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -3,12 +3,59 @@ name: CI/CD - Docker Image from DECS Branch on: pull_request: types: [closed] - branches: [ "develop" ] + branches: [ "main" ] + workflow_dispatch: + inputs: + date_tag: + description: "YYMMDD release tag. Defaults to image-variants.json." + required: false + type: string jobs: + prepare: + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true + runs-on: ubuntu-latest + outputs: + tag_name: ${{ steps.generate_tag.outputs.TAG_NAME }} + matrix: ${{ steps.generate_matrix.outputs.matrix }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Generate release tag + id: generate_tag + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ inputs.date_tag }}" ]]; then + TAG_VERSION="${{ inputs.date_tag }}" + elif [[ -n "${{ github.event.pull_request.head.ref }}" ]]; then + BRANCH_NAME="${{ github.event.pull_request.head.ref }}" + TAG_VERSION="${BRANCH_NAME#decs}" + else + TAG_VERSION="$(python3 -c 'import json; print(json.load(open("image-variants.json", encoding="utf-8"))["default_date_tag"])')" + fi + + if ! [[ "$TAG_VERSION" =~ ^[0-9]{6}$ ]]; then + echo "Release tag must be YYMMDD, got: $TAG_VERSION" >&2 + exit 1 + fi + + echo "TAG_NAME=$TAG_VERSION" >> "$GITHUB_OUTPUT" + + - name: Generate build matrix + id: generate_matrix + run: | + python3 scripts/variant_matrix.py \ + --date-tag "${{ steps.generate_tag.outputs.TAG_NAME }}" \ + --repository dguailab/decs \ + --github-output + build-and-push: - if: github.event.pull_request.merged == true + needs: prepare runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} steps: - name: Checkout repository @@ -20,19 +67,19 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Generate tag from branch name - id: generate_tag - run: | - BRANCH_NAME=${{ github.event.pull_request.head.ref }} - TAG_VERSION=${BRANCH_NAME#decs} - echo "Extracted tag version: $TAG_VERSION" - echo "TAG_NAME=$TAG_VERSION" >> $GITHUB_OUTPUT - - - name: Build and push Docker image + - name: Build and push ${{ matrix.id }} uses: docker/build-push-action@v5 with: context: . + file: ./Dockerfile push: true - tags: | - dguailab/decs:${{ steps.generate_tag.outputs.TAG_NAME }} - dguailab/decs:latest + build-args: | + BASE_IMAGE=${{ matrix.base_image }} + DECS_IMAGE_VARIANT=${{ matrix.id }} + CUDA_VERSION=${{ matrix.cuda_version }} + TENSORFLOW_VERSION=${{ matrix.tensorflow_version }} + TENSORFLOW_PACKAGE=${{ matrix.tensorflow_package }} + PYTHON_VERSION=${{ matrix.python_version }} + UBUNTU_VERSION=${{ matrix.ubuntu_version }} + MIN_NVIDIA_DRIVER=${{ matrix.min_nvidia_driver }} + tags: ${{ matrix.docker_tags }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..43ae0e2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.py[cod] diff --git a/Dockerfile b/Dockerfile index 9b09888..05b95ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,83 +1,114 @@ -# 이 파일은 decs:251002 이미지의 Dockerfile 입니다. -############################# version history ############################# - -##### decs:251002##### -# 변경한 사람: 이소은 -# tensorflow version 2.13.0-gpu -> 2.18.0-gpu - -########################### version history end ########################## - -# TensorFlow 2.18.0 GPU 공식 이미지 사용 (CUDA 12.3, cuDNN 8.9 포함, Ubuntu 22.04 기반) -# https://hub.docker.com/layers/tensorflow/tensorflow/2.18.0-gpu/images/sha256-b076938b81335b8098a58a9e701ea183a652f146419f8601550c000f576e3cc4 -FROM tensorflow/tensorflow:2.18.0-gpu - -# 설치 시 geographic area 를 물어보지 않도록 설정(apt install 시 interrupted 됨) -ENV DEBIAN_FRONTEND noninteractive - -ENV SUDOER_ID svmanager -ENV SUDOER_PW decs2260 -ENV SUDOER_DIR /$SUDOER_ID -ENV SSHD_CONFIG_PATH /etc/ssh/sshd_config +# DECS CUDA/TensorFlow image template. +# Build variants are defined in image-variants.json and passed as build args. + +ARG BASE_IMAGE=nvidia/cuda:12.5.1-cudnn-devel-ubuntu22.04 +FROM ${BASE_IMAGE} + +ARG BASE_IMAGE +ARG DECS_IMAGE_VARIANT=cuda12.5-tf2.20-ubuntu22.04 +ARG CUDA_VERSION=12.5 +ARG TENSORFLOW_VERSION=2.20.0 +ARG TENSORFLOW_PACKAGE=tensorflow==2.20.0 +ARG PYTHON_VERSION=3.10 +ARG UBUNTU_VERSION=22.04 +ARG MIN_NVIDIA_DRIVER=555.42.06 +ARG MINIFORGE_VERSION=25.3.1-0 + +LABEL org.opencontainers.image.base.name="${BASE_IMAGE}" \ + ai.dgu.decs.variant="${DECS_IMAGE_VARIANT}" \ + ai.dgu.decs.cuda="${CUDA_VERSION}" \ + ai.dgu.decs.tensorflow="${TENSORFLOW_VERSION}" \ + ai.dgu.decs.python="${PYTHON_VERSION}" \ + ai.dgu.decs.ubuntu="${UBUNTU_VERSION}" \ + ai.dgu.decs.min_nvidia_driver="${MIN_NVIDIA_DRIVER}" + +ENV DEBIAN_FRONTEND=noninteractive \ + CONDA_DIR=/opt/conda \ + DECS_IMAGE_VARIANT="${DECS_IMAGE_VARIANT}" \ + DECS_CUDA_VERSION="${CUDA_VERSION}" \ + DECS_TENSORFLOW_VERSION="${TENSORFLOW_VERSION}" \ + DECS_PYTHON_VERSION="${PYTHON_VERSION}" \ + DECS_MIN_NVIDIA_DRIVER="${MIN_NVIDIA_DRIVER}" \ + SUDOER_ID=svmanager \ + SUDOER_PW=decs2260 \ + SUDOER_DIR=/svmanager \ + SSHD_CONFIG_PATH=/etc/ssh/sshd_config + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt-get clean \ -&& apt-get -y update \ -&& apt install -y \ -sudo \ -net-tools \ -fcitx-hangul \ -fonts-nanum* \ -vim \ -wget \ -curl \ -ssh \ -software-properties-common - -# motd install -RUN apt-get update && apt-get install -y update-motd - - -# 관리자 계정의 home directory 로 쓸 폴더 추가(home은 nfs이므로, 다른 곳에 생성) -RUN mkdir "$SUDOER_DIR" -# 관리자 계정을 추가, home directory 를 위에서 생성한 폴더로 설정 -RUN useradd -s /bin/bash -d /$SUDOER_ID -G sudo $SUDOER_ID \ - && echo "$SUDOER_ID ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers -# skel 을 복사 (로그인 시 tf-docker 로 뜨지 않게 하는 목적) -RUN cp -R /etc/skel/. "$SUDOER_DIR" - -RUN echo $SUDOER_ID:$SUDOER_PW | chpasswd - - -# decs dir 을 생성 -RUN mkdir /home/decs + && apt-get update \ + && apt-get install -y --no-install-recommends \ + auditd \ + ca-certificates \ + curl \ + dbus-x11 \ + fcitx-hangul \ + fonts-nanum \ + fonts-nanum-coding \ + fonts-nanum-extra \ + gnupg \ + net-tools \ + novnc \ + openssh-server \ + software-properties-common \ + sudo \ + tigervnc-common \ + tigervnc-standalone-server \ + update-motd \ + vim \ + websockify \ + wget \ + xfce4 \ + xfce4-terminal \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p "$SUDOER_DIR" /home/decs /run/sshd \ + && useradd -s /bin/bash -d "$SUDOER_DIR" -G sudo "$SUDOER_ID" \ + && echo "$SUDOER_ID ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \ + && cp -R /etc/skel/. "$SUDOER_DIR" \ + && echo "$SUDOER_ID:$SUDOER_PW" | chpasswd RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ - && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ - && apt-get update && apt-get install -y google-chrome-stable - -RUN printf "LANG=\"ko_KR.UTF-8\"\nLANG=\"ko_KR.EUC-KR\"\nLANGUAGE=\"ko_KR:ko:en_GB:en\"\n" >> /etc/environment \ -&& fc-cache -r - -RUN cat /etc/environment - -# 최신 Anaconda 버전 다운로드 및 설치 -RUN wget https://repo.anaconda.com/archive/Anaconda3-2024.10-1-Linux-x86_64.sh \ - && bash Anaconda3-2024.10-1-Linux-x86_64.sh -b -p /opt/anaconda3 \ - && rm Anaconda3-2024.10-1-Linux-x86_64.sh + && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends google-chrome-stable \ + && rm -rf /var/lib/apt/lists/* + +RUN printf 'LANG="ko_KR.UTF-8"\nLANGUAGE="ko_KR:ko:en_GB:en"\n' >> /etc/environment \ + && fc-cache -r + +RUN wget -q "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-Linux-x86_64.sh" \ + && bash Miniforge3-Linux-x86_64.sh -b -p "$CONDA_DIR" \ + && rm Miniforge3-Linux-x86_64.sh + +ENV PATH=/opt/conda/bin:$PATH + +RUN conda config --system --set channel_priority strict \ + && conda config --system --add channels conda-forge \ + && conda install -n base -y \ + "python=${PYTHON_VERSION}" \ + ipywidgets \ + jupyterlab \ + micromamba \ + notebook \ + pip \ + && python -m pip install --no-cache-dir --upgrade pip \ + && python -m pip install --no-cache-dir "${TENSORFLOW_PACKAGE}" \ + && conda clean -afy \ + && conda init bash + +RUN apt-get update \ + && apt-get install -y --no-install-recommends tigervnc-tools \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /jupyter_config \ + && touch /jupyter_config/jupyter_notebook_config.py -ENV PATH /opt/anaconda3/bin:$PATH -RUN echo "export PATH="/opt/anaconda3/bin:$PATH"" >> /etc/profile \ - && /opt/anaconda3/bin/conda init - -# jupyterlab 설치 -RUN /opt/anaconda3/bin/conda install -y jupyterlab - -# jupyterlab 설정파일 생성 -RUN mkdir /jupyter_config \ - && /opt/anaconda3/bin/jupyter lab --generate-config --config=/jupyter_config/jupyter_notebook_config.py - -# entrypoint.sh 복사 COPY entrypoint.sh / -# SSHD 서버를 실행하고, entrypoint 파일을 start/restart 시 마다 실행, dev/null에 entrypoint 로그를 저장 +# noVNC listens on 6080. The TigerVNC server binds to localhost only. +EXPOSE 6080 + RUN chmod +x /entrypoint.sh ENTRYPOINT ["bash", "/entrypoint.sh"] diff --git a/README.md b/README.md index e189b75..8815743 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,143 @@ -## 🗒️ 이미지 버전 요약 -각 이미지 태그는 생성된 날짜(YYMMDD)를 따르며, latest 태그는 항상 최신 버전의 이미지를 가리킵니다. +## DECS Docker Images -| 이미지 태그 (Image Tag) | TensorFlow 버전 | CUDA / cuDNN | 베이스 OS (Base OS) | 주요 변경사항 및 설명 | -| :--- | :--- | :--- | :--- | :--- | -| `dguailab/decs:latest`
`dguailab/decs:260201` | **2.18.0** | CUDA 12.5
cuDNN 8.9 | Ubuntu 22.04 | **(최신)** 의도되지 않은 MOTD 출력 방지 버그 수정 | -| `dguailab/decs:251023` | **2.18.0** | CUDA 12.5
cuDNN 8.9 | Ubuntu 22.04 | **(이전 안정 버전)** Jupyter Notebook 버전 변경으로 인한 오류 해결 버전 | -| `dguailab/decs:251002` | **2.18.0** | CUDA 12.5
cuDNN 8.9 | Ubuntu 22.04 | TensorFlow 2.18.0 업그레이드, 최신 GPU 환경 지원 | -| `dguailab/decs:250926` | **2.13.0** | CUDA 11.8
cuDNN 8.6 | Ubuntu 20.04 | **(이전 안정 버전)** TensorFlow 2.13.0 기반의 안정화 버전 | +DECS 이미지는 GPU/driver 호환성을 관리하기 위해 CUDA variant별로 빌드한다. 공통 Dockerfile은 하나만 유지하고, 실제 CUDA/TensorFlow 조합은 `image-variants.json`에서 관리한다. -## ⚙️ 사용 방법 -필요한 버전의 이미지를 Docker Hub에서 pull 받아 사용합니다. +## Image Variants -### 최신 버전 사용하기 -latest 태그를 사용하여 항상 최신 버전의 이미지를 받을 수 있습니다. +| Variant tag | CUDA | TensorFlow | Base image | 최소 NVIDIA driver | 상태 | +| --- | --- | --- | --- | --- | --- | +| `cuda11.8-tf2.13-ubuntu22.04-260515` | 11.8 | 2.13.0 | `nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04` | 520.61.05 | stable | +| `cuda12.2-tf2.15-ubuntu22.04-260515` | 12.2 | 2.15.0 | `nvidia/cuda:12.2.2-cudnn8-devel-ubuntu22.04` | 535.104.05 | stable | +| `cuda12.5-tf2.20-ubuntu22.04-260515` | 12.5 | 2.20.0 | `nvidia/cuda:12.5.1-cudnn-devel-ubuntu22.04` | 555.42.06 | stable | +| `cuda12.8-tf2.20-ubuntu22.04-260515` | 12.8 | 2.20.0 | `nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04` | 570.124.06 | experimental | + +Alias tags: + +| Alias | Target | +| --- | --- | +| `latest`, `stable`, `cuda12.5-tf2.20` | CUDA 12.5 / TensorFlow 2.20 | +| `legacy`, `cuda11.8-tf2.13` | CUDA 11.8 / TensorFlow 2.13 | +| `cuda12.2-tf2.15` | CUDA 12.2 / TensorFlow 2.15 | +| `cuda12.8-tf2.20`, `h200-experimental` | CUDA 12.8 / TensorFlow 2.20 | + +TensorFlow 공식 빌드 구성 기준으로 TensorFlow 2.20.0은 CUDA 12.5/cuDNN 9.3 조합이다. CUDA 12.8 이미지는 H200/LAB10 검증 전까지 experimental로 둔다. + +## Included Runtime + +모든 variant는 다음을 포함한다. + +- CUDA/cuDNN base image +- TensorFlow +- system packages: SSH, sudo, auditd, Korean fonts/input, Chrome, Xfce, TigerVNC, noVNC +- Miniforge under `/opt/conda` +- micromamba +- JupyterLab / Notebook / ipywidgets + +`entrypoint.sh`는 시작 시 이미지 variant, CUDA/TensorFlow 버전, 요구 driver 버전, 실제 `nvidia-smi` 정보를 출력한다. `STRICT_CUDA_COMPAT=true`를 주면 host driver가 variant의 최소 driver보다 낮을 때 시작을 실패시킨다. + +## Build + +전체 variant dry-run: + +```bash +python3 scripts/build_variants.py --dry-run ``` -docker pull dguailab/decs:latest + +특정 variant build: + +```bash +python3 scripts/build_variants.py --variant cuda12.5-tf2.20-ubuntu22.04 ``` -### 날짜 태그를 직접 명시하여 받기 +push까지 수행: + +```bash +python3 scripts/build_variants.py --variant cuda12.5-tf2.20-ubuntu22.04 --push ``` -docker pull dguailab/decs:251002 + +GitHub Actions는 `main` 대상 PR이 merge되거나 `workflow_dispatch`로 실행될 때 `image-variants.json`을 읽어 matrix build/push를 수행한다. + +## Usage + +운영 스크립트(`~/uid/script/create_container.sh`)에서는 이미지 이름과 버전을 분리해서 전달한다. + +```bash +bash ~/uid/script/create_container.sh \ + --image decs \ + --version cuda12.5-tf2.20-ubuntu22.04-260515 ``` -### 특정 구버전 사용하기 -이전 버전의 TensorFlow 환경이 필요한 경우, 해당 날짜 태그를 명시하여 이미지를 받습니다. +noVNC는 기존처럼 opt-in이다. + +```bash +--enable-vnc true +``` + +관련 환경변수: + +| 환경변수 | 기본값 | 설명 | +| --- | --- | --- | +| `ENABLE_VNC` | `false` | `true`이면 TigerVNC/noVNC를 시작한다. | +| `VNC_PASSWORD` | 랜덤 8자리 | 지정하지 않으면 `/home/$USER_ID/vnc_password.txt`에 저장한다. | +| `VNC_RESOLUTION` | `1920x1080` | VNC 화면 해상도 | +| `VNC_DEPTH` | `24` | VNC 색상 깊이 | +| `VNC_DISPLAY` | `1` | VNC display 번호. 기본 VNC 포트는 `5901` | +| `NOVNC_PORT` | `6080` | noVNC listen 포트 | +| `STRICT_CUDA_COMPAT` | `false` | 최소 NVIDIA driver 미만이면 startup 실패 | + +## Tests + +이 저장소 내부의 테스트 파일만 사용한다. 외부 `~/uid`와 ansible inventory는 호출 대상이다. + +로컬 이미지 smoke test: + +```bash +python3 scripts/test_image_variants.py --variant cuda12.5-tf2.20-ubuntu22.04 +python3 scripts/test_image_variants.py --variant cuda12.5-tf2.20-ubuntu22.04 --gpu ``` -docker pull dguailab/decs:250926 + +`~/uid/script_test/create_container.sh` dry-run 연동 테스트: + +```bash +python3 scripts/test_uid_create_container.py --variant cuda12.5-tf2.20-ubuntu22.04 +``` + +LAB10 같은 실제 GPU host에서 ansible smoke test: + +```bash +tar -czf /tmp/decs-build-context-260515.tgz Dockerfile entrypoint.sh .dockerignore + +ansible-playbook \ + -i /home/jy/ansible/inventory.ini \ + tests/ansible/decs_image_build.yml \ + -e target_hosts=lab10 \ + -e image_tag=cuda12.8-tf2.20-ubuntu22.04-260515 \ + -e base_image=nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04 \ + -e decs_image_variant=cuda12.8-tf2.20-ubuntu22.04 \ + -e cuda_version=12.8 \ + -e tensorflow_version=2.20.0 \ + -e tensorflow_package=tensorflow==2.20.0 \ + -e min_nvidia_driver=570.124.06 + +ansible-playbook \ + -i /home/jy/ansible/inventory.ini \ + tests/ansible/decs_image_smoke.yml \ + -e target_hosts=lab10 \ + -e image_tag=cuda12.8-tf2.20-ubuntu22.04-260515 ``` +VNC까지 확인: -# ⚒️ 빌드 자동화 -이 Docker 이미지는 GitHub Actions를 통해 자동으로 빌드 및 배포됩니다. -- 실행 조건: decsYYMMDD 형식의 브랜치가 develop 브랜치로 병합(merge)될 때 -- 자동 생성 태그: -- 브랜치 이름에서 추출한 날짜 태그 (예: `dguailab/decs:251002`) -- 최신 버전을 가리키는 latest 태그 (`dguailab/decs:latest`) +```bash +ansible-playbook \ + -i /home/jy/ansible/inventory.ini \ + tests/ansible/decs_image_smoke.yml \ + -e target_hosts=lab10 \ + -e image_tag=cuda12.5-tf2.20-ubuntu22.04-260515 \ + -e enable_vnc=true +``` +## Admin Notes -


-### 🔗 관리자용 노션 문서 링크 +관리자용 노션 문서: https://www.notion.so/DECS-280c7692a263802ca40ff68b38f58dd1?source=copy_link diff --git a/entrypoint.sh b/entrypoint.sh index b03bcea..1089466 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,7 +1,146 @@ #!/bin/bash -sudo apt update -sudo apt install -y auditd +CONDA_DIR="${CONDA_DIR:-/opt/conda}" +JUPYTER_BIN="${JUPYTER_BIN:-$CONDA_DIR/bin/jupyter}" +USER_PW="${USER_PW:-ailab2260}" + +is_truthy() { + case "${1:-}" in + true|TRUE|1|yes|YES|on|ON) return 0 ;; + *) return 1 ;; + esac +} + +version_ge() { + local current="$1" + local required="$2" + [[ "$(printf "%s\n%s\n" "$required" "$current" | sort -V | head -n1)" == "$required" ]] +} + +print_image_runtime_info() { + echo "DECS image variant: ${DECS_IMAGE_VARIANT:-unknown}" + echo "DECS CUDA version: ${DECS_CUDA_VERSION:-unknown}" + echo "DECS TensorFlow version: ${DECS_TENSORFLOW_VERSION:-unknown}" + echo "DECS minimum NVIDIA driver: ${DECS_MIN_NVIDIA_DRIVER:-unknown}" + + if ! command -v nvidia-smi >/dev/null 2>&1; then + echo "nvidia-smi not found. GPU runtime may not be attached." + return 0 + fi + + nvidia-smi --query-gpu=name,driver_version --format=csv,noheader || true + + local required_driver="${DECS_MIN_NVIDIA_DRIVER:-}" + if [[ -z "$required_driver" ]]; then + return 0 + fi + + local host_driver + host_driver="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n1 | tr -d '[:space:]')" + if [[ -z "$host_driver" ]]; then + echo "Could not read NVIDIA driver version from nvidia-smi." + return 0 + fi + + if version_ge "$host_driver" "$required_driver"; then + return 0 + fi + + local message="Host NVIDIA driver $host_driver is lower than required $required_driver for ${DECS_IMAGE_VARIANT:-this image}." + if is_truthy "${STRICT_CUDA_COMPAT:-false}"; then + echo "ERROR: $message" + return 1 + fi + + echo "WARNING: $message Set STRICT_CUDA_COMPAT=true to fail startup." +} + +start_novnc() { + if ! is_truthy "${ENABLE_VNC:-false}"; then + echo "VNC/noVNC disabled. Set ENABLE_VNC=true to enable it." + return 0 + fi + + local user_home="/home/$USER_ID" + local vnc_dir="$user_home/.vnc" + local vnc_display="${VNC_DISPLAY:-1}" + local vnc_resolution="${VNC_RESOLUTION:-1920x1080}" + local vnc_depth="${VNC_DEPTH:-24}" + local novnc_port="${NOVNC_PORT:-6080}" + local vnc_password_file="$user_home/vnc_password.txt" + local vnc_password + + vnc_display="${vnc_display#:}" + if ! [[ "$vnc_display" =~ ^[0-9]+$ && "$novnc_port" =~ ^[0-9]+$ && "$vnc_depth" =~ ^[0-9]+$ ]]; then + echo "Invalid VNC configuration. Check VNC_DISPLAY, NOVNC_PORT, and VNC_DEPTH." + return 1 + fi + local vnc_port=$((5900 + vnc_display)) + + if ! command -v vncserver >/dev/null 2>&1 || ! command -v vncpasswd >/dev/null 2>&1 || ! command -v websockify >/dev/null 2>&1; then + echo "VNC/noVNC packages are not installed. Skipping GUI startup." + return 0 + fi + + mkdir -p "$vnc_dir" "$user_home/decs_jupyter_lab" /tmp/.X11-unix /tmp/.ICE-unix + chown root:root /tmp/.X11-unix /tmp/.ICE-unix + chmod 1777 /tmp/.X11-unix /tmp/.ICE-unix + + if [[ -n "${VNC_PASSWORD:-}" ]]; then + vnc_password="$VNC_PASSWORD" + elif [[ -s "$vnc_password_file" ]]; then + vnc_password=$(tr -d '\r\n' < "$vnc_password_file" | head -c 8) + else + vnc_password=$(tr -dc A-Za-z0-9 "$vnc_password_file" + chmod 600 "$vnc_password_file" + + printf "%s\n" "$vnc_password" | vncpasswd -f > "$vnc_dir/passwd" + chmod 600 "$vnc_dir/passwd" + + cat > "$vnc_dir/xstartup" <<'EOF' +#!/bin/sh +unset SESSION_MANAGER +unset DBUS_SESSION_BUS_ADDRESS +export XDG_SESSION_TYPE=x11 +export XKL_XMODMAP_DISABLE=1 +xrdb "$HOME/.Xresources" 2>/dev/null || true +exec dbus-launch --exit-with-session startxfce4 +EOF + chmod +x "$vnc_dir/xstartup" + chown -R "$USER_ID:$USER_GROUP" "$vnc_dir" "$vnc_password_file" + + sudo -u "$USER_ID" env HOME="$user_home" USER="$USER_ID" \ + vncserver -kill ":$vnc_display" >/tmp/vnc-kill.log 2>&1 || true + + echo "trying TigerVNC on localhost:$vnc_port..." + if ! sudo -u "$USER_ID" env HOME="$user_home" USER="$USER_ID" \ + vncserver -localhost yes ":$vnc_display" -geometry "$vnc_resolution" -depth "$vnc_depth" >/tmp/vncserver.log 2>&1; then + echo "TigerVNC startup failed. See /tmp/vncserver.log." + cat /tmp/vncserver.log + return 1 + fi + echo "TigerVNC listening on localhost:$vnc_port" + + if [[ -d /usr/share/novnc && -f /usr/share/novnc/vnc.html ]]; then + ln -sf /usr/share/novnc/vnc.html /usr/share/novnc/index.html + fi + + pkill -f "websockify.*$novnc_port" >/dev/null 2>&1 || true + echo "trying noVNC on 0.0.0.0:$novnc_port..." + nohup websockify --web=/usr/share/novnc "0.0.0.0:$novnc_port" "localhost:$vnc_port" >/tmp/novnc.log 2>&1 & + echo "noVNC listening on port $novnc_port. VNC password saved to $vnc_password_file" +} + +print_image_runtime_info || exit 1 # /etc/audit/audit.rules 파일에 줄 추가 echo "-a always,exit -F arch=b64 -S unlink -S unlinkat -S rename -S renameat -F auid=$USER_ID -k rm_commands" >> /etc/audit/audit.rules @@ -109,8 +248,11 @@ mkdir -p /home/$USER_ID/.jupyter/ if [ ! -f /home/$USER_ID/.jupyter/jupyter_notebook_config.py ]; then echo "jupyter_notebook_config.py not found, generating..." - /opt/anaconda3/bin/jupyter notebook --generate-config - cp /root/.jupyter/jupyter_notebook_config.py /home/$USER_ID/.jupyter/ + if [ -f /jupyter_config/jupyter_notebook_config.py ]; then + cp /jupyter_config/jupyter_notebook_config.py /home/$USER_ID/.jupyter/jupyter_notebook_config.py + else + "$JUPYTER_BIN" notebook --generate-config --config=/home/$USER_ID/.jupyter/jupyter_notebook_config.py + fi else echo "jupyter_notebook_config.py already exists." fi @@ -126,9 +268,12 @@ chown $USER_ID:$USER_ID /home/$USER_ID/decs_jupyter_lab/jupyter_token.txt # jupyter_lab 기동 echo "trying jupyter lab..." -nohup /opt/anaconda3/bin/jupyter lab --NotebookApp.token=$TOKEN --config=/home/$USER_ID/.jupyter/jupyter_notebook_config.py >/dev/null 2>&1 & +nohup "$JUPYTER_BIN" lab --NotebookApp.token=$TOKEN --config=/home/$USER_ID/.jupyter/jupyter_notebook_config.py >/dev/null 2>&1 & echo "jupyter lab listening!" +# noVNC 기동. 외부에서는 컨테이너의 6080 포트를 매핑해서 접속합니다. +start_novnc || echo "VNC/noVNC startup failed." + # ldconfig permission 오류 방지 # bash.bashrc에서 ldconfig 명령어 삭제 후 명령어 실행 및 결과 출력 sed -i '/ldconfig/d' /etc/bash.bashrc diff --git a/image-variants.json b/image-variants.json new file mode 100644 index 0000000..9c59ab5 --- /dev/null +++ b/image-variants.json @@ -0,0 +1,70 @@ +{ + "repository": "dguailab/decs", + "default_date_tag": "260515", + "notes": [ + "TensorFlow 2.20.0 is officially built against CUDA 12.5 and cuDNN 9.3.", + "The CUDA 12.8 variant is provided as experimental until LAB10/H200 smoke tests pass." + ], + "variants": [ + { + "id": "cuda11.8-tf2.13-ubuntu22.04", + "base_image": "nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04", + "cuda_version": "11.8", + "tensorflow_version": "2.13.0", + "tensorflow_package": "tensorflow==2.13.0", + "python_version": "3.10", + "ubuntu_version": "22.04", + "min_nvidia_driver": "520.61.05", + "support": "stable", + "aliases": [ + "cuda11.8-tf2.13", + "legacy" + ] + }, + { + "id": "cuda12.2-tf2.15-ubuntu22.04", + "base_image": "nvidia/cuda:12.2.2-cudnn8-devel-ubuntu22.04", + "cuda_version": "12.2", + "tensorflow_version": "2.15.0", + "tensorflow_package": "tensorflow==2.15.0", + "python_version": "3.10", + "ubuntu_version": "22.04", + "min_nvidia_driver": "535.104.05", + "support": "stable", + "aliases": [ + "cuda12.2-tf2.15" + ] + }, + { + "id": "cuda12.5-tf2.20-ubuntu22.04", + "base_image": "nvidia/cuda:12.5.1-cudnn-devel-ubuntu22.04", + "cuda_version": "12.5", + "tensorflow_version": "2.20.0", + "tensorflow_package": "tensorflow==2.20.0", + "python_version": "3.10", + "ubuntu_version": "22.04", + "min_nvidia_driver": "555.42.06", + "support": "stable", + "aliases": [ + "cuda12.5-tf2.20", + "stable", + "latest" + ] + }, + { + "id": "cuda12.8-tf2.20-ubuntu22.04", + "base_image": "nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04", + "cuda_version": "12.8", + "tensorflow_version": "2.20.0", + "tensorflow_package": "tensorflow==2.20.0", + "python_version": "3.10", + "ubuntu_version": "22.04", + "min_nvidia_driver": "570.124.06", + "support": "experimental", + "aliases": [ + "cuda12.8-tf2.20", + "h200-experimental" + ] + } + ] +} diff --git a/scripts/build_variants.py b/scripts/build_variants.py new file mode 100755 index 0000000..e70c56d --- /dev/null +++ b/scripts/build_variants.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +import argparse +import json +import shlex +import subprocess +from pathlib import Path + +from variant_matrix import build_tags, load_manifest + + +BUILD_ARG_KEYS = [ + ("BASE_IMAGE", "base_image"), + ("DECS_IMAGE_VARIANT", "id"), + ("CUDA_VERSION", "cuda_version"), + ("TENSORFLOW_VERSION", "tensorflow_version"), + ("TENSORFLOW_PACKAGE", "tensorflow_package"), + ("PYTHON_VERSION", "python_version"), + ("UBUNTU_VERSION", "ubuntu_version"), + ("MIN_NVIDIA_DRIVER", "min_nvidia_driver"), +] + + +def select_variants(manifest, selected): + variants = manifest["variants"] + if not selected: + return variants + matches = [variant for variant in variants if variant["id"] == selected] + if not matches: + raise SystemExit(f"variant not found: {selected}") + return matches + + +def build_command(variant, repository, date_tag, no_cache): + cmd = ["docker", "build", "-f", "Dockerfile"] + if no_cache: + cmd.append("--no-cache") + + for docker_arg, key in BUILD_ARG_KEYS: + cmd.extend(["--build-arg", f"{docker_arg}={variant[key]}"]) + + for tag in build_tags(repository, variant, date_tag): + cmd.extend(["-t", tag]) + + cmd.append(".") + return cmd + + +def main(): + parser = argparse.ArgumentParser(description="Build DECS Docker image variants.") + parser.add_argument("--manifest", default="image-variants.json") + parser.add_argument("--repository") + parser.add_argument("--date-tag") + parser.add_argument("--variant") + parser.add_argument("--push", action="store_true") + parser.add_argument("--no-cache", action="store_true") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + repo_root = Path(__file__).resolve().parents[1] + manifest = load_manifest(repo_root / args.manifest) + repository = args.repository or manifest["repository"] + date_tag = args.date_tag or manifest["default_date_tag"] + + for variant in select_variants(manifest, args.variant): + tags = build_tags(repository, variant, date_tag) + cmd = build_command(variant, repository, date_tag, args.no_cache) + print(shlex.join(cmd)) + if not args.dry_run: + subprocess.run(cmd, cwd=repo_root, check=True) + + if args.push: + for tag in tags: + push_cmd = ["docker", "push", tag] + print(shlex.join(push_cmd)) + if not args.dry_run: + subprocess.run(push_cmd, check=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_image_variants.py b/scripts/test_image_variants.py new file mode 100755 index 0000000..25eb204 --- /dev/null +++ b/scripts/test_image_variants.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +import argparse +import shlex +import subprocess +from pathlib import Path + +from variant_matrix import build_tags, load_manifest + + +CPU_SMOKE = r""" +set -euo pipefail +command -v python +command -v jupyter +command -v micromamba +test -x /entrypoint.sh +python - <<'PY' +import os +import tensorflow as tf + +expected = os.environ.get("DECS_TENSORFLOW_VERSION") +actual = tf.__version__.split("+", 1)[0] +print("tensorflow", tf.__version__) +print("expected", expected) +if expected and actual != expected: + raise SystemExit(f"TensorFlow version mismatch: {actual} != {expected}") +PY +jupyter --version +micromamba --version +""" + +GPU_SMOKE = r""" +set -euo pipefail +nvidia-smi +python - <<'PY' +import tensorflow as tf + +gpus = tf.config.list_physical_devices("GPU") +print("tensorflow", tf.__version__) +print("gpus", gpus) +if not gpus: + raise SystemExit("TensorFlow did not detect a GPU") +PY +""" + + +def select_variants(manifest, selected): + if not selected: + return manifest["variants"] + matches = [variant for variant in manifest["variants"] if variant["id"] == selected] + if not matches: + raise SystemExit(f"variant not found: {selected}") + return matches + + +def run(cmd, dry_run): + print(shlex.join(cmd)) + if not dry_run: + subprocess.run(cmd, check=True) + + +def main(): + parser = argparse.ArgumentParser(description="Run local smoke tests for DECS image variants.") + parser.add_argument("--manifest", default="image-variants.json") + parser.add_argument("--repository") + parser.add_argument("--date-tag") + parser.add_argument("--variant") + parser.add_argument("--gpu", action="store_true", help="Require GPU visibility through Docker.") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + repo_root = Path(__file__).resolve().parents[1] + manifest = load_manifest(repo_root / args.manifest) + repository = args.repository or manifest["repository"] + date_tag = args.date_tag or manifest["default_date_tag"] + + for variant in select_variants(manifest, args.variant): + image = build_tags(repository, variant, date_tag)[0] + run(["docker", "image", "inspect", image], args.dry_run) + + cmd = ["docker", "run", "--rm", "--entrypoint", "bash"] + if args.gpu: + cmd.extend(["--gpus", "all"]) + cmd.extend([image, "-lc", CPU_SMOKE]) + run(cmd, args.dry_run) + + if args.gpu: + run(["docker", "run", "--rm", "--gpus", "all", "--entrypoint", "bash", image, "-lc", GPU_SMOKE], args.dry_run) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_uid_create_container.py b/scripts/test_uid_create_container.py new file mode 100755 index 0000000..08b5c8f --- /dev/null +++ b/scripts/test_uid_create_container.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +import argparse +import re +import shlex +import subprocess +from datetime import date, timedelta +from pathlib import Path + +from variant_matrix import build_tags, load_manifest + + +def select_variants(manifest, selected): + if not selected: + return manifest["variants"] + matches = [variant for variant in manifest["variants"] if variant["id"] == selected] + if not matches: + raise SystemExit(f"variant not found: {selected}") + return matches + + +def safe_name(value): + return re.sub(r"[^a-zA-Z0-9]", "", value).lower()[:24] + + +def main(): + parser = argparse.ArgumentParser( + description="Exercise ~/uid/script_test/create_container.sh with DECS image tags." + ) + parser.add_argument("--manifest", default="image-variants.json") + parser.add_argument("--repository", default="dguailab/decs") + parser.add_argument("--date-tag") + parser.add_argument("--variant") + parser.add_argument("--uid-root", default="/home/jy/uid") + parser.add_argument("--domain", default="LAB") + parser.add_argument("--server-number", default="10") + parser.add_argument("--created-by", default="decs-test") + parser.add_argument("--email", default="decs-smoke@example.invalid") + parser.add_argument("--phone", default="000-0000-0000") + parser.add_argument("--enable-vnc", action="store_true") + parser.add_argument("--print-only", action="store_true") + args = parser.parse_args() + + repo_root = Path(__file__).resolve().parents[1] + manifest = load_manifest(repo_root / args.manifest) + date_tag = args.date_tag or manifest["default_date_tag"] + uid_script = Path(args.uid_root) / "script_test" / "create_container.sh" + if not uid_script.exists(): + raise SystemExit(f"uid dry-run wrapper not found: {uid_script}") + + repository_name = args.repository.rsplit("/", 1)[-1] + expiration = (date.today() + timedelta(days=7)).isoformat() + + for variant in select_variants(manifest, args.variant): + tag = build_tags(args.repository, variant, date_tag)[0].split(":", 1)[1] + username = f"decs{safe_name(variant['id'])}" + cmd = [ + str(uid_script), + "--name", + "DECS Smoke Test", + "--username", + username, + "--no-group", + "--domain", + args.domain, + "--server-number", + str(args.server_number), + "--expiration-date", + expiration, + "--image", + repository_name, + "--version", + tag, + "--no-container-name", + "--no-additional-ports", + "--created-by", + args.created_by, + "--email", + args.email, + "--phone", + args.phone, + "--note", + f"DECS image dry-run smoke for {variant['id']}", + "--user-password", + "DecsTest1234", + ] + if args.enable_vnc: + cmd.extend(["--enable-vnc", "true", "--vnc-password", "DecsVNC1"]) + + print(shlex.join(cmd)) + if not args.print_only: + subprocess.run(cmd, check=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/variant_matrix.py b/scripts/variant_matrix.py new file mode 100755 index 0000000..4dcf69b --- /dev/null +++ b/scripts/variant_matrix.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +from pathlib import Path + + +def load_manifest(path): + with Path(path).open(encoding="utf-8") as f: + manifest = json.load(f) + + seen = set() + for variant in manifest["variants"]: + variant_id = variant["id"] + if variant_id in seen: + raise SystemExit(f"duplicate variant id: {variant_id}") + seen.add(variant_id) + return manifest + + +def build_tags(repository, variant, date_tag): + tags = [f"{repository}:{variant['id']}-{date_tag}"] + tags.extend(f"{repository}:{alias}" for alias in variant.get("aliases", [])) + return tags + + +def build_matrix(manifest, repository, date_tag, selected_variant=None): + include = [] + for variant in manifest["variants"]: + if selected_variant and variant["id"] != selected_variant: + continue + + item = dict(variant) + item["docker_tags"] = "\n".join(build_tags(repository, variant, date_tag)) + include.append(item) + + if not include: + raise SystemExit(f"variant not found: {selected_variant}") + + return {"include": include} + + +def write_github_output(name, value): + output_path = os.environ.get("GITHUB_OUTPUT") + if not output_path: + print(f"{name}={value}") + return + + with Path(output_path).open("a", encoding="utf-8") as f: + if "\n" in value: + f.write(f"{name}< + rm -rf {{ remote_build_dir | quote }} + && mkdir -p {{ remote_build_dir | quote }} + + - name: Upload build context archive + ansible.builtin.copy: + src: "{{ build_context_archive }}" + dest: "{{ build_context_archive }}" + mode: "0644" + + - name: Extract build context archive + ansible.builtin.shell: > + tar -xzf {{ build_context_archive | quote }} + -C {{ remote_build_dir | quote }} + + - name: Build DECS image variant + ansible.builtin.shell: | + set -euo pipefail + docker build \ + -f Dockerfile \ + --build-arg BASE_IMAGE={{ base_image | quote }} \ + --build-arg DECS_IMAGE_VARIANT={{ decs_image_variant | quote }} \ + --build-arg CUDA_VERSION={{ cuda_version | quote }} \ + --build-arg TENSORFLOW_VERSION={{ tensorflow_version | quote }} \ + --build-arg TENSORFLOW_PACKAGE={{ tensorflow_package | quote }} \ + --build-arg PYTHON_VERSION={{ python_version | quote }} \ + --build-arg UBUNTU_VERSION={{ ubuntu_version | quote }} \ + --build-arg MIN_NVIDIA_DRIVER={{ min_nvidia_driver | quote }} \ + -t {{ image_repository }}:{{ image_tag }} \ + . + args: + chdir: "{{ remote_build_dir }}" + executable: /bin/bash diff --git a/tests/ansible/decs_image_smoke.yml b/tests/ansible/decs_image_smoke.yml new file mode 100644 index 0000000..e02ddcf --- /dev/null +++ b/tests/ansible/decs_image_smoke.yml @@ -0,0 +1,102 @@ +--- +- name: Smoke test a DECS image on GPU Docker hosts + hosts: "{{ target_hosts | default('lab10') }}" + gather_facts: false + vars: + image_repository: dguailab/decs + image_tag: cuda12.5-tf2.20-ubuntu22.04-260515 + test_container_name: "decs-smoke-{{ image_tag | regex_replace('[^A-Za-z0-9_.-]', '-') }}" + test_home_root: /tmp/decs-smoke-home + test_username: decstest + test_group: decstest + test_uid: "31000" + test_gid: "31000" + test_memory: 8g + enable_vnc: false + strict_cuda_compat: true + + tasks: + - name: Run smoke lifecycle + block: + - name: Prepare smoke home mount + ansible.builtin.shell: "mkdir -p {{ test_home_root | quote }}" + + - name: Ensure image exists locally or can be pulled + ansible.builtin.shell: > + docker image inspect {{ image_repository }}:{{ image_tag }} >/dev/null 2>&1 + || docker pull {{ image_repository }}:{{ image_tag }} + + - name: Remove old smoke container + ansible.builtin.shell: "docker rm -f {{ test_container_name | quote }} >/dev/null 2>&1 || true" + + - name: Run smoke container with create_container.sh-compatible options + ansible.builtin.shell: > + docker run -dit + --gpus device=all + --memory={{ test_memory }} + --memory-swap={{ test_memory }} + --runtime=nvidia + --cap-add=SYS_ADMIN + --ipc=host + --mount type=bind,source={{ test_home_root | quote }},target=/home/ + --name {{ test_container_name | quote }} + -e USER_ID={{ test_username | quote }} + -e GID={{ test_gid | quote }} + -e USER_PW='DecsTest1234' + -e USER_GROUP={{ test_group | quote }} + -e UID={{ test_uid | quote }} + -e ENABLE_VNC={{ (enable_vnc | bool) | ternary('true', 'false') | quote }} + -e STRICT_CUDA_COMPAT={{ (strict_cuda_compat | bool) | ternary('true', 'false') | quote }} + -e NVIDIA_DRIVER_CAPABILITIES='compute,utility,graphics,display' + {{ image_repository }}:{{ image_tag }} + + - name: Wait for entrypoint to create user assets + ansible.builtin.shell: > + timeout 90 bash -lc + 'until docker exec {{ test_container_name | quote }} test -f /home/{{ test_username }}/decs_jupyter_lab/jupyter_token.txt; + do sleep 3; done' + + - name: Validate shell, conda, Jupyter, TensorFlow, and GPU visibility + ansible.builtin.shell: | + set -euo pipefail + docker exec {{ test_container_name | quote }} bash -lc ' + set -euo pipefail + id {{ test_username | quote }} + test -x /entrypoint.sh + command -v python + command -v jupyter + command -v micromamba + nvidia-smi + python - <<'"'"'PY'"'"' + import tensorflow as tf + print("tensorflow", tf.__version__) + gpus = tf.config.list_physical_devices("GPU") + print("gpus", gpus) + if not gpus: + raise SystemExit("TensorFlow did not detect a GPU") + PY + ' + args: + executable: /bin/bash + + - name: Validate noVNC process when enabled + ansible.builtin.shell: > + docker exec {{ test_container_name | quote }} + bash -lc 'test -f /home/{{ test_username }}/vnc_password.txt && pgrep -f websockify' + when: enable_vnc | bool + + always: + - name: Show smoke container logs + ansible.builtin.shell: "docker logs --tail 120 {{ test_container_name | quote }}" + register: smoke_logs + changed_when: false + failed_when: false + + - name: Print smoke logs + ansible.builtin.debug: + var: smoke_logs.stdout_lines + + - name: Cleanup smoke container + ansible.builtin.shell: "docker rm -f {{ test_container_name | quote }} >/dev/null 2>&1 || true" + changed_when: false + failed_when: false