Use uv and pyproject.toml for python services

.github/images.yml

@@ -1,45 +1,45 @@
 images:
   - path: ./images/gh-gl-sync
-    image: ghcr.io/spack/ci-bridge:0.0.51
+    image: ghcr.io/spack/ci-bridge:0.0.52
 
   - path: ./images/ci-key-clear
-    image: ghcr.io/spack/ci-key-clear:0.0.5
+    image: ghcr.io/spack/ci-key-clear:0.0.6
 
   - path: ./images/gitlab-stuckpods
     image: ghcr.io/spack/stuckpods:0.0.2
 
   - path: ./images/gitlab-clear-pipelines
-    image: ghcr.io/spack/gitlab-clear-pipelines:0.0.1
+    image: ghcr.io/spack/gitlab-clear-pipelines:0.0.2
 
   - path: ./images/gitlab-delete-stale-branches
-    image: ghcr.io/spack/gitlab-delete-stale-branches:0.0.2
+    image: ghcr.io/spack/gitlab-delete-stale-branches:0.0.3
 
   - path: ./images/gitlab-skipped-pipelines
-    image: ghcr.io/spack/gitlab-skipped-pipelines:0.0.3
+    image: ghcr.io/spack/gitlab-skipped-pipelines:0.0.4
 
   - path: ./images/notary
     image: ghcr.io/spack/notary:0.0.4
 
   - path: ./images/python-aws-bash
-    image: ghcr.io/spack/python-aws-bash:0.0.2
+    image: ghcr.io/spack/python-aws-bash:0.0.3
 
   - path: ./images/snapshot-release-tags
-    image: ghcr.io/spack/snapshot-release-tags:0.0.6
+    image: ghcr.io/spack/snapshot-release-tags:0.0.7
 
   - path: ./images/cache-indexer
-    image: ghcr.io/spack/cache-indexer:0.0.6
+    image: ghcr.io/spack/cache-indexer:0.0.7
 
   - path: ./analytics
-    image: ghcr.io/spack/django:0.5.25
+    image: ghcr.io/spack/django:0.5.26
 
   - path: ./images/ci-prune-buildcache
-    image: ghcr.io/spack/ci-prune-buildcache:0.0.8
+    image: ghcr.io/spack/ci-prune-buildcache:0.0.9
 
   - path: ./images/protected-publish
-    image: ghcr.io/spack/protected-publish:0.0.9
+    image: ghcr.io/spack/protected-publish:0.0.10
 
   - path: ./images/retry-trigger-jobs
-    image: ghcr.io/spack/retry-trigger-jobs:0.0.2
+    image: ghcr.io/spack/retry-trigger-jobs:0.0.3
 
   - path: ./images/runner-node-certificate-controller
     image: ghcr.io/spack/runner-node-certificate-controller:0.0.3

.github/renovate.json5

@@ -117,7 +117,6 @@
     },
     {
       matchManagers: [
-        "pip_requirements",
         "pip-compile",
         "pep621",
         "poetry",
@@ -174,7 +173,6 @@
     },
     {
       matchManagers: [
-        "pip_requirements",
         "pip-compile",
         "pep621",
         "poetry",

.github/workflows/spack_ci_bridge.yml

@@ -19,7 +19,9 @@ jobs:
           git                                                                  \
           openssh-client                                                       \
           python3-pip
-        sudo pip3 install -r images/gh-gl-sync/requirements.txt
+        pip3 install uv
+        uv sync --project images/gh-gl-sync --frozen
+        echo "$PWD/images/gh-gl-sync/.venv/bin" >> $GITHUB_PATH
     - name: test
       run: |
         ctest -VV -S .github/spack_ci_bridge.cmake

analytics/Dockerfile

@@ -15,17 +15,21 @@ RUN apt-get update && \
 RUN git clone -c feature.manyFiles=true --depth 1 --branch v0.22.0 https://github.com/spack/spack.git /opt/spack
 
 # Include spack import paths for python packages. Order is important
-ENV PYTHONPATH "/opt/spack/lib/spack:$PYTHONPATH"
-ENV PYTHONPATH "/opt/spack/lib/spack/external/_vendoring:$PYTHONPATH"
-ENV PYTHONPATH "/opt/spack/lib/spack/external:$PYTHONPATH"
+ENV PYTHONPATH="/opt/spack/lib/spack"
+ENV PYTHONPATH="/opt/spack/lib/spack/external/_vendoring:${PYTHONPATH}"
+ENV PYTHONPATH="/opt/spack/lib/spack/external:${PYTHONPATH}"
 
 
+# Install dependencies into the image's system prefix rather than a project .venv
+ENV UV_PROJECT_ENVIRONMENT=/usr/local
+
 WORKDIR /app
 
-COPY requirements.txt /app/requirements.txt
+COPY pyproject.toml uv.lock /app/
 
 RUN pip install --upgrade uv
-RUN uv pip install --system -r requirements.txt
+# --no-install-project: the app itself is not installed, it's run from /app below
+RUN uv sync --frozen --no-install-project --no-cache
 
 COPY . .
 

analytics/dev/django.Dockerfile

@@ -15,26 +15,29 @@ RUN apt-get update && \
 # Cleanup
 RUN rm -rf /var/lib/apt/lists/*
 
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
-
-# Only copy the setup.py, it will still force all install_requires to be installed,
-# but find_packages() will find nothing (which is fine). When Docker Compose mounts the real source
-# over top of this directory, the .egg-link in site-packages resolves to the mounted directory
-# and all package modules are importable.
-COPY ./setup.py /opt/django-project/setup.py
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+# Only copy the dependency metadata, so that the layer is cached independently of the source.
+# --no-install-project keeps the app itself out of site-packages; Docker Compose mounts the real
+# source over /opt/django-project and PYTHONPATH (set below) makes its modules importable.
+ENV UV_PROJECT_ENVIRONMENT=/usr/local
+COPY ./pyproject.toml ./uv.lock /opt/django-project/
 RUN pip install --upgrade uv
-RUN uv pip install --system --editable /opt/django-project[dev]
+RUN uv sync --project /opt/django-project --frozen --no-install-project --extra dev --no-cache
 
 
 # Install spack
 RUN git clone -c feature.manyFiles=true https://github.com/spack/spack.git /opt/spack
 RUN cd /opt/spack && git checkout v0.22.0
 
 # Include spack import paths for python packages. Order is important
-ENV PYTHONPATH "/opt/spack/lib/spack:$PYTHONPATH"
-ENV PYTHONPATH "/opt/spack/lib/spack/external/_vendoring:$PYTHONPATH"
-ENV PYTHONPATH "/opt/spack/lib/spack/external:$PYTHONPATH"
+ENV PYTHONPATH="/opt/spack/lib/spack"
+ENV PYTHONPATH="/opt/spack/lib/spack/external/_vendoring:${PYTHONPATH}"
+ENV PYTHONPATH="/opt/spack/lib/spack/external:${PYTHONPATH}"
+
+# Make the bind-mounted project source importable
+ENV PYTHONPATH="/opt/django-project:${PYTHONPATH}"
 
 
 # Use a directory name which will never be an import name, as isort considers this as first-party.

analytics/pyproject.toml

@@ -1,3 +1,66 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "analytics"
+version = "0.1.0"
+description = ""
+readme = "README.md"
+license = "Apache-2.0"
+authors = [
+    { name = "Kitware, Inc.", email = "kitware@kitware.com" },
+]
+keywords = []
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Web Environment",
+    "Framework :: Django :: 5.1",
+    "Framework :: Django",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python",
+]
+requires-python = ">=3.13"
+dependencies = [
+    "cachetools==5.5.0",
+    "celery[redis]==5.4.0",
+    "dj-database-url==2.3.0",
+    "dj-email-url==1.0.6",
+    "django-click==2.4.0",
+    "django-configurations[database,email]==2.5.1",
+    "django-cors-headers==4.6.0",
+    "django-extensions==3.2.3",
+    "django-girder-utils==0.13.1",
+    "django-ninja==1.3.0",
+    "django-s3-file-field[minio]==1.0.1",
+    "django==5.1.15",
+    "gunicorn==23.0.0",
+    "opensearch-dsl==2.1.0",
+    "kubernetes==31.0.0",
+    "sentry-sdk[django,pure_eval]==2.19.2",
+    "rich==13.9.4",
+    "psycopg2-binary==2.9.10",
+    "python-gitlab==5.3.0",
+    "pyyaml==6.0.2",
+    "requests==2.32.4",
+    "tqdm==4.66.3",
+    "whitenoise[brotli]==6.8.2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "types-cachetools",
+    "django-debug-toolbar",
+    "ipython",
+    "tox",
+]
+
+[tool.setuptools.packages.find]
+include = ["analytics*"]
+
 [tool.black]
 line-length = 100
 target-version = ["py313"]