Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 34 additions & 1 deletion .github/workflows/ci-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@ on:
description: 'Disable buildx cache'
required: false
default: 'false'
run-postinstall-checks:
description: 'Run feature postInstallCheck commands in built images'
required: false
default: 'false'
type: choice
options:
- 'true'
- 'false'

permissions:
contents: read
Expand Down Expand Up @@ -74,13 +82,38 @@ jobs:
echo "$profile" >> profiles-built.txt
done

- name: Run post-install contract checks
env:
RUN_POSTINSTALL_CHECKS: ${{ github.event.inputs['run-postinstall-checks'] || 'false' }}
run: |
set -uo pipefail
if [ "$RUN_POSTINSTALL_CHECKS" != 'true' ]; then
echo "Post-install contract checks disabled"
exit 0
fi

failed=0
while read -r profile; do
[ -n "$profile" ] || continue
tag=$(sed -n "/^target \"final-$profile\" /,/^}/p" generated/docker-bake.hcl \
| awk '/tags = \[/{flag=1; next} flag && /\]/{exit} flag {gsub(/[",]/, ""); gsub(/^[ \t]+|[ \t]+$/, ""); if ($0 != "") {print; exit}}')
if [ -z "$tag" ]; then
echo "No tag found in bake file for $profile; skipping post-install checks" >&2
continue
fi
echo "Running post-install contract checks for $tag"
docker run --rm --entrypoint /opt/solen/run-postinstall-checks.sh "$tag" || failed=1
done < profiles-built.txt

exit "$failed"

- name: Generate SBOM and Trivy scans
run: |
set -uo pipefail
while read -r profile; do
[ -n "$profile" ] || continue
tag=$(sed -n "/^target \"final-$profile\" /,/^}/p" generated/docker-bake.hcl \
| sed -n '/tags = \[/,/\]/p' | grep -m1 '"' | tr -d '"')
| awk '/tags = \[/{flag=1; next} flag && /\]/{exit} flag {gsub(/[",]/, ""); gsub(/^[ \t]+|[ \t]+$/, ""); if ($0 != "") {print; exit}}')
if [ -z "$tag" ]; then
echo "No tag found in bake file for $profile; skipping scans" >&2
continue
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci-validate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,4 @@ jobs:
run: python -m pytest tests/ -q

- name: Lint
run: ruff check solen-cli
run: ruff check solen-cli tests
32 changes: 28 additions & 4 deletions checksums.json
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@
}
},
"quarto": {
"pinned_version": "1.8.24",
"pinned_version": "1.10.18",
"archive_format": "tar.gz / deb / pkg / msi",
"archive_url_template": "https://github.com/quarto-dev/quarto-cli/releases/download/v{version}/quarto-{version}-linux-{arch}.tar.gz",
"checksum_asset": "quarto-{version}-checksums.txt",
Expand All @@ -131,6 +131,10 @@
"1.8.24": {
"amd64": "6b83c1c9b6f2ce6454798b42260bd2ee184551d74debe817b8aaf28b09ac22d0",
"arm64": "89a97a65a242a5b9b010a9f9978928c1d8e4ac02a558c9cd91a110c3f2611fdd"
},
"1.10.18": {
"amd64": "afad071b5bd22c02f2d300695743189d3650e0537a53073e654b630cff2b0c73",
"arm64": "f6a07df68e25330b5df34f65d3df66bca605acce3b830c593a58e91884d4cf6c"
}
}
},
Expand All @@ -147,14 +151,18 @@
}
,
"ijava": {
"pinned_version": "1.4.5",
"pinned_version": "1.4.6-pr12",
"archive_format": "zip",
"archive_filename": "IJava-latest.zip",
"archive_url_template": "https://github.com/ebpro/IJava/releases/download/v{version}/IJava-latest.zip",
"archive_filename": "IJava-{version}.zip",
"archive_url_template": "https://github.com/ebpro/IJava/releases/download/v{version}/IJava-{version}.zip",
"notes": "IJava distribution zip produced by CI (contains install.py and JAR).",
"checksums": {
"1.4.5": {
"amd64": "8336fbcf26cc570fbf33ee52a2e6051d865214355654e32218467db49658f2b5"
},
"1.4.6-pr12": {
"amd64": "c2fa3bcb3fc22602e7e116bf0f2590dcc521ebf8cecb347885b799919ab6a870",
"arm64": "c2fa3bcb3fc22602e7e116bf0f2590dcc521ebf8cecb347885b799919ab6a870"
}
}
}
Expand All @@ -175,6 +183,22 @@
}
}
,
"tinytex": {
"pinned_version": "2026.09",
"archive_format": "tar.gz",
"archive_url_template": "https://github.com/rstudio/tinytex-releases/releases/download/v{version}/installer-unix-v{version}.tar.gz",
"latest_version_cmd": "curl -s https://api.github.com/repos/rstudio/tinytex-releases/releases/latest | jq -r '.tag_name' | sed 's/^v//'",
"notes": "TinyTeX unix installer; same asset for amd64/arm64.",
"checksums": {
"2026.09": {
"amd64": "aaf6e0115b1419bafe45c368ae41f46abe024d21556c989db17707698bf69080",
"arm64": "aaf6e0115b1419bafe45c368ae41f46abe024d21556c989db17707698bf69080",
"x86_64": "aaf6e0115b1419bafe45c368ae41f46abe024d21556c989db17707698bf69080",
"aarch64": "aaf6e0115b1419bafe45c368ae41f46abe024d21556c989db17707698bf69080"
}
}
}
,
"miniforge": {
"pinned_version": "25.11.0-1",
"archive_format": "sh",
Expand Down
237 changes: 237 additions & 0 deletions docs/TESTING.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
# Testing

This document describes the current testing strategy for Solen and the post-install contract checks added for feature validation.

## Test layers

Solen testing is split into several layers:

1. **Static validation**
- feature schema validation
- profile generation
- version synchronization
- Python unit tests
- linting

2. **Feature contract tests**
- each feature can declare a `postInstallCheck`
- generated Dockerfiles can run those checks during or after image build
- checks are intended to verify that a feature is usable in the final container

3. **Feature test scripts**
- features may include `*test*.sh` scripts
- scripts are staged into the image under `/opt/solen/feature-tests/<feature>/`
- `/opt/solen/run-feature-tests.sh` executes all staged feature tests

4. **CI build validation**
- canonical builds run in GitHub Actions with Docker BuildKit
- post-install checks can be enabled per build run

## Local setup

Create a virtual environment and install the CLI with development dependencies:

```bash
python3 -m venv .venv
.venv/bin/pip install --upgrade pip
.venv/bin/pip install -e 'solen-cli[dev]'
```

Optional, for local mypy checks:

```bash
.venv/bin/pip install types-PyYAML
```

## Static validation

Run these commands from the repository root:

```bash
.venv/bin/python -m pytest tests -q
.venv/bin/ruff check solen-cli tests
.venv/bin/solen validate features
.venv/bin/solen versions sync --check
```

Generate profiles locally:

```bash
.venv/bin/solen generate profiles --matrix profiles/matrix --out /tmp/generated-profiles --chain
```

For local work, prefer generating profiles into a temporary directory rather than directly into `generated/profiles/`.

## Unit tests

The `tests/` directory contains pytest-based validation for generator and feature contract helpers.

Current relevant tests include:

- `tests/test_postinstall_checks.py`
- `tests/test_postinstall_check_collection.py`

Run them with:

```bash
.venv/bin/python -m pytest tests -q
```

## Feature post-install checks

A feature can declare a post-install check in `feature.json`:

```json
{
"id": "java-kernel",
"postInstallCheck": {
"command": "bash -lc '...'",
"description": "Verify Java Jupyter kernelspec is registered"
}
}
```

The generated Dockerfile embeds a runner at:

```text
/opt/solen/run-postinstall-checks.sh
```

Behavior:

- commands are base64-encoded to avoid shell escaping issues
- checks run as `${NB_USER:-jovyan}`
- the runner prefers `runuser`, then `setpriv`, then `su`
- all configured checks are executed
- the runner exits with the number of failed checks

The behavior is controlled by:

```dockerfile
ARG RUN_POSTINSTALL_CHECKS="false"
ENV RUN_POSTINSTALL_CHECKS="${RUN_POSTINSTALL_CHECKS}"
```

When enabled, the generated image also:

- runs per-feature post-install checks immediately after each feature install
- runs the full post-install check runner near the end of the install stage

## Feature test scripts

Features may include test scripts such as:

```text
features/java-kernel/test-java-kernel.sh
```

Conventions:

- use bash
- fail fast with `set -euo pipefail`
- verify the runtime artifacts that the feature is supposed to provide
- avoid assumptions about login-shell state where possible
- target the non-root user when relevant

Example:

```bash
#!/usr/bin/env bash
set -euo pipefail

java_bin="$(command -v java || true)"
if [ -z "$java_bin" ]; then
java_bin="/home/${NB_USER:-jovyan}/.sdkman/candidates/java/current/bin/java"
fi

if [ ! -x "$java_bin" ]; then
echo "ERROR: java runtime not found" >&2
exit 1
fi

"$java_bin" -version
```

Generated images stage these scripts into:

```text
/opt/solen/feature-tests/<feature>/
```

and provide:

```text
/opt/solen/run-feature-tests.sh
```

To run all staged feature tests in a built image:

```bash
docker run --rm --entrypoint /opt/solen/run-feature-tests.sh <image>
```

## CI validation

### `ci-validate.yml`

The validation workflow runs static checks, including:

- `solen validate features`
- profile generation
- devcontainer generation
- version synchronization
- pytest
- ruff over `solen-cli` and `tests`

### `ci-build.yml`

The build workflow:

- generates profiles
- generates `generated/Dockerfile`
- generates `generated/docker-bake.hcl`
- builds the selected profiles
- optionally runs post-install checks in the built images

The post-install check gate is controlled by the workflow input:

```yaml
run-postinstall-checks:
description: 'Run feature postInstallCheck commands in built images'
required: false
default: 'false'
type: choice
options:
- 'true'
- 'false'
```

When enabled, CI runs:

```bash
docker run --rm --entrypoint /opt/solen/run-postinstall-checks.sh <image>
```

for each built profile.

## Local limitations

Some validations are easier or only possible in CI:

- canonical image builds require Docker BuildKit and `docker buildx bake`
- feature runtime checks require the built image
- Java-related checks require a Java-enabled profile
- Jupyter/kernel checks require the relevant Python/Jupyter stack

If local Docker/BuildKit support is unavailable, rely on CI for image-level validation.

## Roadmap

Future testing work should extend the current slice with:

- E2E/BDD scenarios for full profile builds
- automated smoke tests for `quarto-full`
- optional SonarQube quality gates
- Harbor image admission checks
- public registry validation
- richer test reporting for CI artifacts
7 changes: 6 additions & 1 deletion features/build-essentials/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ else
fi

# This feature installs build tools and monitoring utilities.
PKGS="build-essential cmake pkg-config python3-dev libssl-dev libffi-dev git htop lsof strace"
PKGS="build-essential cmake pkg-config python3-dev libssl-dev libffi-dev git htop lsof strace fd-find ripgrep"

echo "build-essentials: installing packages: ${PKGS}"
if command -v apt_install >/dev/null 2>&1; then
Expand All @@ -30,4 +30,9 @@ else
fi
rm -rf /var/lib/apt/lists/* || true

# Ubuntu/Debian package `fd-find` installs `fdfind`, while consumers expect `fd`.
if [ -x /usr/bin/fdfind ] && [ ! -e /usr/local/bin/fd ]; then
ln -sf /usr/bin/fdfind /usr/local/bin/fd || true
fi

echo "build-essentials: done"
2 changes: 1 addition & 1 deletion features/buildah/feature.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
],
"options": {},
"postInstallCheck": {
"command": "buildah --version",
"command": "sh -c 'case \"$(uname -m)\" in x86_64|amd64) exit 0;; *) buildah --version;; esac'",
"description": "Verify buildah is installed"
},
Comment on lines 20 to 23
"documentationURL": "./features/buildah/README.md"
Expand Down
Loading