docker-compose.yml

services:
  llama:
    build: .
    container_name: diffusiongemma
    restart: unless-stopped
    ports:
      - "${PORT:-8000}:8080"
    environment:
      - HF_TOKEN=${HF_TOKEN}
    volumes:
      # Reuse a host HF cache so the GGUF is downloaded once across rebuilds /
      # restarts. Override HF_CACHE_HOST in `.env` to point at any local dir.
      - ${HF_CACHE_HOST:-./hf-cache}:/root/.cache/huggingface
    command:
      - "-hf"
      - "${MODEL_REPO:-unsloth/diffusiongemma-26B-A4B-it-GGUF}:${MODEL_QUANT:-Q4_K_M}"
      - "-c"
      - "${CONTEXT_LENGTH:-32768}"
      - "-ngl"
      - "${GPU_LAYERS:-99}"
      - "--host"
      - "0.0.0.0"
      - "--port"
      - "8080"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: ${GPU_COUNT:-1}
              capabilities: [gpu]