diff --git a/.github/actions/setup-nix/action.yml b/.github/actions/setup-nix/action.yml new file mode 100644 index 000000000..a99a548cd --- /dev/null +++ b/.github/actions/setup-nix/action.yml @@ -0,0 +1,47 @@ +name: Nix Build +description: Install Nix, configure Cachix, and build flake targets. + +inputs: + build: + description: Flake output namespace, such as packages or checks. + required: true + system: + description: Nix system to build for. + required: true + targets: + description: Newline-separated package or check targets to build. + required: true + cachix_auth_token: + description: Cachix write token. + required: true + +runs: + using: composite + steps: + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@main + with: + extra-conf: | + accept-flake-config = true + + - name: Set up Cachix + uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17 + with: + name: openshell + authToken: ${{ inputs.cachix_auth_token }} + skipAddingSubstituter: true + + - name: Build targets + shell: bash + env: + BUILD: ${{ inputs.build }} + SYSTEM: ${{ inputs.system }} + TARGETS: ${{ inputs.targets }} + run: | + attrs=() + while IFS= read -r target; do + [ -n "$target" ] || continue + attrs+=(".#${BUILD}.${SYSTEM}.${target}") + done <<< "$TARGETS" + + nix build "${attrs[@]}" --no-link --no-update-lock-file diff --git a/.github/workflows/e2e-label-help.yml b/.github/workflows/e2e-label-help.yml deleted file mode 100644 index 1190bcd3d..000000000 --- a/.github/workflows/e2e-label-help.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: E2E Label Help - -# When an E2E label is applied, post a PR comment -# telling the maintainer the next manual step. We don't dispatch the workflow -# ourselves: a workflow_dispatch-triggered run does not surface in the PR's -# Checks tab, so we'd lose in-progress visibility. Instead we point the -# maintainer at either the existing run (re-run from the UI) or the -# `/ok to test ` command needed to refresh the mirror. -# -# Uses `pull_request_target` so forked PRs get a token capable of posting -# comments. The job never checks out PR code; it only calls the GitHub API. - -on: - pull_request_target: - types: [labeled] - -permissions: {} - -jobs: - hint: - name: Post next-step hint for E2E label - if: github.event.label.name == 'test:e2e' || github.event.label.name == 'test:e2e-gpu' || github.event.label.name == 'test:e2e-kubernetes' - runs-on: ubuntu-latest - permissions: - pull-requests: write - actions: read - contents: read - steps: - - name: Post comment - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - LABEL_NAME: ${{ github.event.label.name }} - shell: bash - run: | - set -euo pipefail - - workflow_file=branch-e2e.yml - workflow_name="Branch E2E Checks" - case "$LABEL_NAME" in - test:e2e) - suite_summary="the standard E2E suite" - build_summary="gateway and supervisor images" - status_summary="The matching required CI gate status on this PR will flip green automatically once the run finishes." - ;; - test:e2e-gpu) - suite_summary="GPU E2E" - build_summary="supervisor image" - status_summary="The matching required CI gate status on this PR will flip green automatically once the run finishes." - ;; - test:e2e-kubernetes) - suite_summary="Kubernetes HA E2E" - build_summary="gateway and supervisor images" - status_summary="This is an optional proof-of-life suite; failures are visible in the workflow run but do not publish a required CI gate status." - ;; - *) echo "Unrecognized label $LABEL_NAME"; exit 1 ;; - esac - - mirror_ref="pull-request/$PR_NUMBER" - mirror_sha=$(gh api "repos/$GH_REPO/branches/$mirror_ref" --jq '.commit.sha' 2>/dev/null || echo "") - short_pr=${PR_HEAD_SHA:0:7} - - if [ -z "$mirror_sha" ]; then - body="Label \`$LABEL_NAME\` applied, but \`$mirror_ref\` does not exist yet. A maintainer needs to comment \`/ok to test $PR_HEAD_SHA\` to mirror this PR. Once the mirror exists, re-apply the label or re-run [$workflow_name](https://github.com/$GH_REPO/actions/workflows/$workflow_file) from the Actions tab." - elif [ "$mirror_sha" != "$PR_HEAD_SHA" ]; then - short_mirror=${mirror_sha:0:7} - body="Label \`$LABEL_NAME\` applied, but \`$mirror_ref\` is at \`$short_mirror\` while the PR head is \`$short_pr\`. A maintainer needs to comment \`/ok to test $PR_HEAD_SHA\` to refresh the mirror. Once the mirror catches up, re-run [$workflow_name](https://github.com/$GH_REPO/actions/workflows/$workflow_file) from the Actions tab." - else - run_id=$(gh api "repos/$GH_REPO/actions/workflows/$workflow_file/runs?head_sha=$PR_HEAD_SHA&event=push" \ - --jq '.workflow_runs | sort_by(.created_at) | reverse | .[0].id // empty') - if [ -n "$run_id" ]; then - instructions="Open [the existing run](https://github.com/$GH_REPO/actions/runs/$run_id) and click **Re-run all jobs** to execute with the label set." - else - workflow_link="[$workflow_name](https://github.com/$GH_REPO/actions/workflows/$workflow_file)" - instructions="Open $workflow_link, find the run for commit \`$short_pr\`, and click **Re-run all jobs** to execute with the label set." - fi - body="Label \`$LABEL_NAME\` applied for \`$short_pr\`. $instructions The run will execute $suite_summary after building the required $build_summary once. $status_summary" - fi - - gh pr comment "$PR_NUMBER" --body "$body" diff --git a/.github/workflows/nix-ci.yml b/.github/workflows/nix-ci.yml new file mode 100644 index 000000000..d45a99313 --- /dev/null +++ b/.github/workflows/nix-ci.yml @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Nix CI + +on: + push: + branches: + - main + - "pull-request/[0-9]+" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + build: + name: Build (${{ matrix.target.system }}) + runs-on: ${{ matrix.target.runner }} + strategy: + fail-fast: false + matrix: + target: + - system: x86_64-linux + runner: linux-amd64-cpu8 + - system: aarch64-linux + runner: linux-arm64-cpu8 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Build packages + uses: ./.github/actions/setup-nix + with: + build: packages + system: ${{ matrix.target.system }} + targets: | + openshell + openshell-gateway + openshell-sandbox + openshell-driver-kubernetes + openshell-driver-podman + openshell-driver-vm + cachix_auth_token: ${{ secrets.CACHIX_AUTH_TOKEN }} + + images: + name: Build Images (${{ matrix.target.system }}) + needs: build + runs-on: ${{ matrix.target.runner }} + strategy: + fail-fast: false + matrix: + target: + - system: x86_64-linux + runner: linux-amd64-cpu8 + - system: aarch64-linux + runner: linux-arm64-cpu8 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Build images + uses: ./.github/actions/setup-nix + with: + build: packages + system: ${{ matrix.target.system }} + targets: | + openshell-gateway-image + openshell-supervisor-image + cachix_auth_token: ${{ secrets.CACHIX_AUTH_TOKEN }} + + test: + name: Test (${{ matrix.target.system }}) + needs: build + runs-on: ${{ matrix.target.runner }} + strategy: + fail-fast: false + matrix: + target: + - system: x86_64-linux + runner: linux-amd64-cpu8 + - system: aarch64-linux + runner: linux-arm64-cpu8 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Run tests + uses: ./.github/actions/setup-nix + with: + build: checks + system: ${{ matrix.target.system }} + targets: | + openshell-bootstrap-test + openshell-cli-test + openshell-core-test + openshell-driver-docker-test + openshell-driver-kubernetes-test + openshell-driver-podman-test + openshell-driver-vm-test + openshell-ocsf-test + openshell-policy-test + openshell-prover-test + openshell-providers-test + openshell-router-test + openshell-sandbox-test + openshell-server-macros-test + openshell-server-test + openshell-tui-test + openshell-vfio-test + cachix_auth_token: ${{ secrets.CACHIX_AUTH_TOKEN }} + + lint: + name: Lint + runs-on: linux-amd64-cpu8 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Run lints + uses: ./.github/actions/setup-nix + with: + build: checks + system: x86_64-linux + targets: | + rustfmt + spdx-headers + openshell-bootstrap-clippy + openshell-cli-clippy + openshell-core-clippy + openshell-driver-docker-clippy + openshell-driver-kubernetes-clippy + openshell-driver-podman-clippy + openshell-driver-vm-clippy + openshell-ocsf-clippy + openshell-policy-clippy + openshell-prover-clippy + openshell-providers-clippy + openshell-router-clippy + openshell-sandbox-clippy + openshell-server-macros-clippy + openshell-server-clippy + openshell-tui-clippy + openshell-vfio-clippy + cachix_auth_token: ${{ secrets.CACHIX_AUTH_TOKEN }} diff --git a/.github/workflows/required-ci-gates.yml b/.github/workflows/required-ci-gates.yml deleted file mode 100644 index ca068cf5c..000000000 --- a/.github/workflows/required-ci-gates.yml +++ /dev/null @@ -1,233 +0,0 @@ -name: Required CI Gates - -on: - pull_request_target: - types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] - workflow_run: - workflows: - - Branch Checks - - Branch E2E Checks - - Helm Lint - types: [completed] - -permissions: - actions: read - contents: read - pull-requests: read - statuses: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.workflow_run.head_sha || github.run_id }} - cancel-in-progress: true - -jobs: - publish: - name: Publish required CI gate statuses - runs-on: ubuntu-latest - steps: - - name: Evaluate required CI gates - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - EVENT_NAME: ${{ github.event_name }} - PR_NUMBER_FROM_EVENT: ${{ github.event.pull_request.number }} - PR_HEAD_SHA_FROM_EVENT: ${{ github.event.pull_request.head.sha }} - PR_LABELS_FROM_EVENT: ${{ toJSON(github.event.pull_request.labels.*.name) }} - WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} - WORKFLOW_RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - WORKFLOW_RUN_EVENT: ${{ github.event.workflow_run.event }} - shell: bash - run: | - set -euo pipefail - - post_status() { - local context="$1" - local state="$2" - local description="$3" - local target_url="${4:-}" - - args=( - --method POST - "repos/$GH_REPO/statuses/$HEAD_SHA" - -f "state=$state" - -f "context=$context" - -f "description=$description" - ) - if [ -n "$target_url" ]; then - args+=(-f "target_url=$target_url") - fi - - echo "$context: $state - $description" - gh api "${args[@]}" >/dev/null - } - - has_label() { - local label="$1" - jq -e --arg label "$label" 'index($label) != null' <<< "$LABELS_JSON" >/dev/null - } - - resolve_pull_request_event() { - PR_NUMBER="$PR_NUMBER_FROM_EVENT" - HEAD_SHA="$PR_HEAD_SHA_FROM_EVENT" - LABELS_JSON=$(jq -c . <<< "$PR_LABELS_FROM_EVENT") - } - - load_pr_context() { - PR_NUMBER="$1" - - local pr state - pr=$(gh api "repos/$GH_REPO/pulls/$PR_NUMBER") - state=$(jq -r '.state' <<< "$pr") - if [ "$state" != "open" ]; then - echo "PR #$PR_NUMBER is $state; nothing to publish." - exit 0 - fi - - HEAD_SHA=$(jq -r '.head.sha' <<< "$pr") - LABELS_JSON=$(gh api "repos/$GH_REPO/issues/$PR_NUMBER" --jq '[.labels[].name]') - } - - resolve_workflow_run_event() { - if [ "$WORKFLOW_RUN_EVENT" != "push" ]; then - echo "Ignoring workflow_run from event '$WORKFLOW_RUN_EVENT'." - exit 0 - fi - - if [[ "$WORKFLOW_RUN_HEAD_BRANCH" =~ ^pull-request/([0-9]+)$ ]]; then - load_pr_context "${BASH_REMATCH[1]}" - return - fi - - local associated_prs pr - associated_prs=$(gh api "repos/$GH_REPO/commits/$WORKFLOW_RUN_HEAD_SHA/pulls") - pr=$(jq -c 'map(select(.state == "open"))[0] // empty' <<< "$associated_prs") - if [ -z "$pr" ]; then - echo "No open PR associated with $WORKFLOW_RUN_HEAD_SHA; nothing to publish." - exit 0 - fi - - load_pr_context "$(jq -r '.number' <<< "$pr")" - } - - resolve_context() { - if [ "$EVENT_NAME" = "pull_request_target" ]; then - resolve_pull_request_event - elif [ "$EVENT_NAME" = "workflow_run" ]; then - resolve_workflow_run_event - else - echo "Unsupported event '$EVENT_NAME'." - exit 1 - fi - - PR_URL="https://github.com/$GH_REPO/pull/$PR_NUMBER" - MIRROR_REF="pull-request/$PR_NUMBER" - } - - verify_mirror() { - local context="$1" - local mirror_sha - - mirror_sha=$(gh api "repos/$GH_REPO/branches/$MIRROR_REF" --jq '.commit.sha' 2>/dev/null || true) - if [ -z "$mirror_sha" ]; then - post_status "$context" pending "Waiting for /ok to test mirror" "$PR_URL" - return 1 - fi - - if [ "$mirror_sha" != "$HEAD_SHA" ]; then - post_status "$context" pending "Waiting for /ok to test mirror" "$PR_URL" - return 1 - fi - - return 0 - } - - evaluate_workflow() { - local context="$1" - local workflow_file="$2" - local workflow_name="$3" - local required_label="${4:-}" - local required_job_name="${5:-}" - local workflow_url="https://github.com/$GH_REPO/actions/workflows/$workflow_file" - - if [ -n "$required_label" ] && ! has_label "$required_label"; then - post_status "$context" success "$required_label not applied" "$PR_URL" - return 0 - fi - - if ! verify_mirror "$context"; then - return 0 - fi - - local runs latest run_id status conclusion run_url real_success - runs=$(gh api "repos/$GH_REPO/actions/workflows/$workflow_file/runs?head_sha=$HEAD_SHA&event=push" --jq '.workflow_runs') - latest=$(jq -c --arg branch "$MIRROR_REF" '[.[] | select(.head_branch == $branch)] | sort_by(.created_at) | reverse | .[0] // empty' <<< "$runs") - - if [ -z "$latest" ]; then - post_status "$context" pending "Waiting for $workflow_name" "$workflow_url" - return 0 - fi - - run_id=$(jq -r '.id' <<< "$latest") - status=$(jq -r '.status' <<< "$latest") - conclusion=$(jq -r '.conclusion' <<< "$latest") - run_url=$(jq -r '.html_url' <<< "$latest") - - if [ "$status" != "completed" ]; then - post_status "$context" pending "$workflow_name is $status" "$run_url" - return 0 - fi - - if [ -n "$required_job_name" ]; then - local jobs required_job job_status job_conclusion - jobs=$(gh api "repos/$GH_REPO/actions/runs/$run_id/jobs?per_page=100" --jq '.jobs') - required_job=$(jq -c --arg name "$required_job_name" '[.[] | select(.name == $name)] | .[0] // empty' <<< "$jobs") - - if [ -z "$required_job" ]; then - if [ "$conclusion" = "success" ]; then - post_status "$context" pending "Waiting for $required_job_name" "$run_url" - else - post_status "$context" failure "$required_job_name did not run" "$run_url" - fi - return 0 - fi - - job_status=$(jq -r '.status' <<< "$required_job") - job_conclusion=$(jq -r '.conclusion' <<< "$required_job") - - if [ "$job_status" != "completed" ]; then - post_status "$context" pending "$required_job_name is $job_status" "$run_url" - return 0 - fi - - if [ "$job_conclusion" = "success" ]; then - post_status "$context" success "$required_job_name passed" "$run_url" - elif [ "$job_conclusion" = "skipped" ] && [ "$conclusion" = "success" ]; then - post_status "$context" pending "Waiting for $required_job_name" "$run_url" - else - post_status "$context" failure "$required_job_name concluded $job_conclusion" "$run_url" - fi - return 0 - fi - - if [ "$conclusion" != "success" ]; then - post_status "$context" failure "$workflow_name concluded $conclusion" "$run_url" - return 0 - fi - - real_success=$(gh api "repos/$GH_REPO/actions/runs/$run_id/jobs?per_page=100" \ - --jq '[.jobs[] | select(.conclusion == "success" and .name != "Resolve PR metadata")] | length') - - if [ "$real_success" -lt 1 ]; then - post_status "$context" failure "No real CI jobs ran" "$run_url" - return 0 - fi - - post_status "$context" success "$workflow_name passed" "$run_url" - } - - resolve_context - - evaluate_workflow "OpenShell / Branch Checks" "branch-checks.yml" "Branch Checks" - evaluate_workflow "OpenShell / E2E" "branch-e2e.yml" "Branch E2E Checks" "test:e2e" "Core E2E result" - evaluate_workflow "OpenShell / GPU E2E" "branch-e2e.yml" "Branch E2E Checks" "test:e2e-gpu" "GPU E2E result" - evaluate_workflow "OpenShell / Helm Lint" "helm-lint.yml" "Helm Lint" diff --git a/.gitignore b/.gitignore index a3d613775..048511ad8 100644 --- a/.gitignore +++ b/.gitignore @@ -225,3 +225,6 @@ rfc.md # Markdown/mermaid lint tooling deps scripts/lint-mermaid/node_modules/ + +# Nix +result* diff --git a/crates/openshell-bootstrap/Cargo.toml b/crates/openshell-bootstrap/Cargo.toml index c860cb138..f57550209 100644 --- a/crates/openshell-bootstrap/Cargo.toml +++ b/crates/openshell-bootstrap/Cargo.toml @@ -9,6 +9,10 @@ license.workspace = true repository.workspace = true rust-version.workspace = true +[lib] +name = "openshell_bootstrap" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core", default-features = false } bollard = "0.20" diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index bf3581164..c4a417ed1 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_core" +path = "src/lib.rs" + [dependencies] prost = { workspace = true } prost-types = { workspace = true } diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 7955772a6..6740fc064 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -12,8 +12,12 @@ fn main() -> Result<(), Box> { // builds where .git is absent, this silently does nothing and the binary // falls back to CARGO_PKG_VERSION (which is already sed-patched by the // build pipeline). - println!("cargo:rerun-if-changed=../../.git/HEAD"); - println!("cargo:rerun-if-changed=../../.git/refs/tags"); + if Path::new("../../.git/HEAD").exists() { + println!("cargo:rerun-if-changed=../../.git/HEAD"); + } + if Path::new("../../.git/refs/tags").exists() { + println!("cargo:rerun-if-changed=../../.git/refs/tags"); + } if let Some(version) = git_version() { println!("cargo:rustc-env=OPENSHELL_GIT_VERSION={version}"); @@ -22,15 +26,16 @@ fn main() -> Result<(), Box> { // --- Protobuf compilation --- // Re-run when anything under proto/ changes (including newly added .proto files). println!("cargo:rerun-if-changed={PROTO_REL}"); - // Use bundled protoc from protobuf-src. The system protoc (from apt-get) - // does not bundle the well-known type includes (google/protobuf/struct.proto - // etc.), so we must use protobuf-src which ships both the binary and the - // include tree. - // SAFETY: This is run at build time in a single-threaded build script context. - // No other threads are reading environment variables concurrently. - #[allow(unsafe_code)] - unsafe { - env::set_var("PROTOC", protobuf_src::protoc()); + if env::var_os("PROTOC").is_none() && !path_has_protoc() { + // Keep non-Nix builds working without requiring users to install protoc. + // Nix builds provide protoc explicitly, so they do not rely on this + // vendored fallback. + // SAFETY: This is run at build time in a single-threaded build script context. + // No other threads are reading environment variables concurrently. + #[allow(unsafe_code)] + unsafe { + env::set_var("PROTOC", protobuf_src::protoc()); + } } let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); @@ -72,6 +77,16 @@ fn collect_proto_files(dir: &Path, out: &mut Vec) -> std::io::Result<() Ok(()) } +fn path_has_protoc() -> bool { + let Some(path) = env::var_os("PATH") else { + return false; + }; + + env::split_paths(&path) + .map(|dir| dir.join(format!("protoc{}", env::consts::EXE_SUFFIX))) + .any(|candidate| candidate.is_file()) +} + /// Derive a version string from `git describe --tags`. /// /// Implements the "guess-next-dev" convention used by the release pipeline diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 7e1bc069c..60660b521 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_driver_docker" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core", default-features = false } diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 07fa91015..2bd378b11 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_driver_kubernetes" +path = "src/lib.rs" + [[bin]] name = "openshell-driver-kubernetes" path = "src/main.rs" diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index ed798c0ab..d3e2464a9 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_driver_podman" +path = "src/lib.rs" + [[bin]] name = "openshell-driver-podman" path = "src/main.rs" diff --git a/crates/openshell-ocsf/Cargo.toml b/crates/openshell-ocsf/Cargo.toml index 14cc93ba3..fca761bd1 100644 --- a/crates/openshell-ocsf/Cargo.toml +++ b/crates/openshell-ocsf/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_ocsf" +path = "src/lib.rs" + [dependencies] chrono = { version = "0.4", features = ["serde"] } serde = { workspace = true } diff --git a/crates/openshell-policy/Cargo.toml b/crates/openshell-policy/Cargo.toml index 16719de13..216f01459 100644 --- a/crates/openshell-policy/Cargo.toml +++ b/crates/openshell-policy/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_policy" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core", default-features = false } serde = { workspace = true } diff --git a/crates/openshell-prover/Cargo.toml b/crates/openshell-prover/Cargo.toml index ee815f3a3..749c05379 100644 --- a/crates/openshell-prover/Cargo.toml +++ b/crates/openshell-prover/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_prover" +path = "src/lib.rs" + [features] bundled-z3 = ["z3/bundled"] diff --git a/crates/openshell-providers/Cargo.toml b/crates/openshell-providers/Cargo.toml index 9b294d7b7..dc10e2e10 100644 --- a/crates/openshell-providers/Cargo.toml +++ b/crates/openshell-providers/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_providers" +path = "src/lib.rs" + [dependencies] glob = { workspace = true } openshell-core = { path = "../openshell-core", default-features = false } diff --git a/crates/openshell-router/Cargo.toml b/crates/openshell-router/Cargo.toml index 97bbf4dc7..ffdd3378d 100644 --- a/crates/openshell-router/Cargo.toml +++ b/crates/openshell-router/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_router" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core", default-features = false } bytes = { workspace = true } diff --git a/crates/openshell-server-macros/Cargo.toml b/crates/openshell-server-macros/Cargo.toml index f929d43a6..fc04db568 100644 --- a/crates/openshell-server-macros/Cargo.toml +++ b/crates/openshell-server-macros/Cargo.toml @@ -10,6 +10,8 @@ license.workspace = true repository.workspace = true [lib] +name = "openshell_server_macros" +path = "src/lib.rs" proc-macro = true [dependencies] diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 71febf0af..33610cfc6 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -10,6 +10,9 @@ license.workspace = true repository.workspace = true rust-version.workspace = true +[lib] +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core" } openshell-ocsf = { path = "../openshell-ocsf" } diff --git a/crates/openshell-supervisor-network/src/procfs.rs b/crates/openshell-supervisor-network/src/procfs.rs index 3ac8dbe14..3f2de8d34 100644 --- a/crates/openshell-supervisor-network/src/procfs.rs +++ b/crates/openshell-supervisor-network/src/procfs.rs @@ -508,6 +508,17 @@ mod tests { use super::*; use std::io::Write; + #[cfg(target_os = "linux")] + fn find_on_path(name: &str) -> PathBuf { + std::env::var_os("PATH") + .and_then(|paths| { + std::env::split_paths(&paths) + .map(|dir| dir.join(name)) + .find(|path| path.is_file()) + }) + .unwrap_or_else(|| panic!("{name} not found on PATH")) + } + /// Block until `/proc//exe` points at `target`. `Command::spawn` returns /// once the child is scheduled, not once it has completed `exec()`; on /// contended runners the readlink can still show the parent (test harness) @@ -602,10 +613,12 @@ mod tests { fn binary_path_strips_deleted_suffix() { use std::os::unix::fs::PermissionsExt; - // Copy /bin/sleep to a temp path we control so we can unlink it. + // Copy a shell to a temp path we control so we can unlink it. Nix + // coreutils dispatches by argv[0], so a copied `sleep` exits when + // renamed to these test filenames. let tmp = tempfile::TempDir::new().unwrap(); let exe_path = tmp.path().join("deleted-sleep"); - std::fs::copy("/bin/sleep", &exe_path).unwrap(); + std::fs::copy(find_on_path("sh"), &exe_path).unwrap(); std::fs::set_permissions(&exe_path, std::fs::Permissions::from_mode(0o755)).unwrap(); // Spawn a child from the temp binary, then unlink it while the @@ -613,7 +626,7 @@ mod tests { // `/proc//exe`, but readlink will now return the tainted // " (deleted)" string. let mut cmd = std::process::Command::new(&exe_path); - cmd.arg("5"); + cmd.args(["-c", "sleep 5; :"]); let mut child = spawn_retrying_on_etxtbsy(&mut cmd); let pid: i32 = child.id().cast_signed(); wait_for_child_exec(pid, &exe_path); @@ -649,6 +662,7 @@ mod tests { /// must be returned unchanged — we only strip when `stat()` reports /// the raw readlink target missing. This guards against the trusted /// identity source misattributing a running binary to a truncated + /// /// sibling path. #[cfg(target_os = "linux")] #[test] @@ -659,11 +673,11 @@ mod tests { // Basename literally ends with " (deleted)" while the file is still // on disk — a pathological but legal filename. let exe_path = tmp.path().join("sleepy (deleted)"); - std::fs::copy("/bin/sleep", &exe_path).unwrap(); + std::fs::copy(find_on_path("sh"), &exe_path).unwrap(); std::fs::set_permissions(&exe_path, std::fs::Permissions::from_mode(0o755)).unwrap(); let mut cmd = std::process::Command::new(&exe_path); - cmd.arg("5"); + cmd.args(["-c", "sleep 5; :"]); let mut child = spawn_retrying_on_etxtbsy(&mut cmd); let pid: i32 = child.id().cast_signed(); wait_for_child_exec(pid, &exe_path); @@ -702,11 +716,11 @@ mod tests { raw_name.extend_from_slice(b".bin"); let exe_path = tmp.path().join(OsString::from_vec(raw_name)); - std::fs::copy("/bin/sleep", &exe_path).unwrap(); + std::fs::copy(find_on_path("sh"), &exe_path).unwrap(); std::fs::set_permissions(&exe_path, std::fs::Permissions::from_mode(0o755)).unwrap(); let mut cmd = std::process::Command::new(&exe_path); - cmd.arg("5"); + cmd.args(["-c", "sleep 5; :"]); let mut child = spawn_retrying_on_etxtbsy(&mut cmd); let pid: i32 = child.id().cast_signed(); wait_for_child_exec(pid, &exe_path); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 691382469..3530a12a9 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -6664,9 +6664,10 @@ network_policies: #[tokio::test] async fn test_resolve_check_allowed_ips_rejects_outside_allowlist() { - // 8.8.8.8 resolves to a public IP which is NOT in 10.0.0.0/8 + // A resolved public IP outside 10.0.0.0/8 must be rejected. let nets = parse_allowed_ips(&["10.0.0.0/8".to_string()]).unwrap(); - let result = resolve_and_check_allowed_ips("dns.google", 443, &nets, 0).await; + let addrs = ["8.8.8.8:443".parse().unwrap()]; + let result = validate_allowed_ips_for_resolved_addrs("dns.google", 443, &addrs, &nets); assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -7330,14 +7331,13 @@ network_policies: #[tokio::test] async fn test_forward_public_ip_allowed_without_allowed_ips() { - // Public IPs (e.g. dns.google -> 8.8.8.8) should pass through - // resolve_and_reject_internal without needing allowed_ips. - let result = resolve_and_reject_internal("dns.google", 80, 0).await; + // Public resolved IPs should pass through without needing allowed_ips. + let addrs = ["8.8.8.8:80".parse().unwrap()]; + let result = reject_internal_resolved_addrs("dns.google", &addrs); assert!( result.is_ok(), "Public IP should be allowed without allowed_ips: {result:?}" ); - let addrs = result.unwrap(); assert!(!addrs.is_empty(), "Should resolve to at least one address"); // All resolved addresses should be public. for addr in &addrs { diff --git a/crates/openshell-supervisor-process/Cargo.toml b/crates/openshell-supervisor-process/Cargo.toml index b2dad859e..08b1d315e 100644 --- a/crates/openshell-supervisor-process/Cargo.toml +++ b/crates/openshell-supervisor-process/Cargo.toml @@ -10,6 +10,9 @@ license.workspace = true repository.workspace = true rust-version.workspace = true +[lib] +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core" } openshell-ocsf = { path = "../openshell-ocsf" } diff --git a/crates/openshell-supervisor-process/src/child_env.rs b/crates/openshell-supervisor-process/src/child_env.rs index 32eecbee3..8ba836203 100644 --- a/crates/openshell-supervisor-process/src/child_env.rs +++ b/crates/openshell-supervisor-process/src/child_env.rs @@ -47,7 +47,7 @@ mod tests { #[test] fn apply_proxy_env_includes_node_proxy_opt_in_and_local_bypass() { - let mut cmd = Command::new("/usr/bin/env"); + let mut cmd = Command::new("env"); cmd.stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::null()); @@ -67,7 +67,7 @@ mod tests { #[test] fn apply_tls_env_sets_node_and_bundle_paths() { - let mut cmd = Command::new("/usr/bin/env"); + let mut cmd = Command::new("env"); cmd.stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::null()); diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 9f9fe1822..d19303fba 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -1310,7 +1310,7 @@ mod tests { #[tokio::test] async fn inject_provider_env_sets_placeholder_values() { - let mut cmd = Command::new("/usr/bin/env"); + let mut cmd = Command::new("env"); cmd.stdin(StdStdio::null()) .stdout(StdStdio::piped()) .stderr(StdStdio::null()); diff --git a/crates/openshell-tui/Cargo.toml b/crates/openshell-tui/Cargo.toml index 238166136..cff3ea9c7 100644 --- a/crates/openshell-tui/Cargo.toml +++ b/crates/openshell-tui/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_tui" +path = "src/lib.rs" + [dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-bootstrap = { path = "../openshell-bootstrap" } diff --git a/crates/openshell-vfio/Cargo.toml b/crates/openshell-vfio/Cargo.toml index b6d7cc3cd..7752d4543 100644 --- a/crates/openshell-vfio/Cargo.toml +++ b/crates/openshell-vfio/Cargo.toml @@ -10,6 +10,10 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +name = "openshell_vfio" +path = "src/lib.rs" + [dependencies] serde = { workspace = true } serde_json = { workspace = true } diff --git a/flake.lock b/flake.lock index 7b9881771..8de4f5362 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,20 @@ { "nodes": { + "crane": { + "locked": { + "lastModified": 1780099841, + "narHash": "sha256-EVZd2RsbpreRUDSi9rBwPY+ZxoyMaiEBbZxxhljbaS4=", + "owner": "ipetkov", + "repo": "crane", + "rev": "0532eb17955225173906d671fb36306bdeb1e2dc", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -36,6 +51,7 @@ }, "root": { "inputs": { + "crane": "crane", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", "rust-overlay": "rust-overlay", diff --git a/flake.nix b/flake.nix index 13c4857bc..e920da975 100644 --- a/flake.nix +++ b/flake.nix @@ -4,6 +4,13 @@ { description = "OpenShell development environment"; + nixConfig = { + extra-substituters = [ "https://openshell.cachix.org" ]; + extra-trusted-public-keys = [ + "openshell.cachix.org-1:OAr5MunsfH5PZvUsfD08OtGx5RtcwdNZGJdU5FqLm5w=" + ]; + }; + inputs = { flake-utils.url = "github:numtide/flake-utils"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; @@ -11,6 +18,7 @@ url = "github:oxalica/rust-overlay"; inputs.nixpkgs.follows = "nixpkgs"; }; + crane.url = "github:ipetkov/crane"; treefmt-nix = { url = "github:numtide/treefmt-nix"; inputs.nixpkgs.follows = "nixpkgs"; @@ -22,6 +30,7 @@ flake-utils, nixpkgs, rust-overlay, + crane, treefmt-nix, ... }: @@ -32,22 +41,99 @@ inherit system; overlays = [ (import rust-overlay) ]; }; + lib = pkgs.lib; rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; + + craneLib = (crane.mkLib pkgs).overrideToolchain (_: rustToolchain); + + crateSpecs = import ./nix/crate.nix { + inherit pkgs; + root = ./.; + }; + + # Crate-by-crate crane helpers (workspace graph, minimal per-crate + # source, buildWorkspaceCrate). See nix/workspace.nix. + workspace = import ./nix/workspace.nix { + inherit lib pkgs craneLib; + root = ./.; + inherit crateSpecs; + }; + inherit (workspace) buildWorkspaceCrate; + + workspaceCrates = lib.mapAttrs (_: buildWorkspaceCrate) crateSpecs; + crates = { + openshell = workspaceCrates.openshell-cli.package; + openshell-gateway = workspaceCrates.openshell-server.package; + openshell-sandbox = workspaceCrates.openshell-sandbox.package; + openshell-driver-vm = workspaceCrates.openshell-driver-vm.package; + openshell-driver-kubernetes = workspaceCrates.openshell-driver-kubernetes.package; + openshell-driver-podman = workspaceCrates.openshell-driver-podman.package; + }; + + images = + if pkgs.stdenv.hostPlatform.isLinux then + import ./nix/images.nix { + inherit pkgs; + gateway = crates.openshell-gateway; + supervisor = crates.openshell-sandbox; + } + else + { }; + + crateTests = lib.mapAttrs' ( + name: crate: lib.nameValuePair "${name}-test" crate.test + ) workspaceCrates; + crateClippy = lib.mapAttrs' ( + name: crate: lib.nameValuePair "${name}-clippy" crate.clippy + ) workspaceCrates; + treefmtEval = treefmt-nix.lib.evalModule pkgs { projectRootFile = "flake.nix"; programs.nixfmt.enable = true; }; + + spdxHeaders = pkgs.runCommand "spdx-headers" { src = lib.cleanSource ./.; } '' + cd "$src" + ${pkgs.python3}/bin/python scripts/update_license_headers.py --check + touch "$out" + ''; in { + packages = + crates + // images + // { + default = pkgs.symlinkJoin { + name = "openshell-0.0.0"; + paths = lib.attrValues crates; + }; + }; + + checks = + crateTests + // crateClippy + // { + rustfmt = craneLib.cargoFmt { + pname = "openshell-workspace"; + src = craneLib.cleanCargoSource ./.; + cargoExtraArgs = "--all"; + }; + spdx-headers = spdxHeaders; + }; + devShells.default = pkgs.mkShell { packages = with pkgs; [ rustToolchain # Required to find packages pkg-config + # Required for protobuf code generation. + protobuf # Required for bindgen generation. llvmPackages.libclang # system dependency for openshell-prover z3 + # caching utility + cachix ]; env = { diff --git a/nix/crate.nix b/nix/crate.nix new file mode 100644 index 000000000..8b4b22128 --- /dev/null +++ b/nix/crate.nix @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ + pkgs, + root, +}: +{ + # Each crate declares the compile-time assets and build tools it needs. The + # workspace builder collects nativeBuildInputs/buildInputs/env from the + # transitive Cargo closure. + openshell-bootstrap = { + dir = "openshell-bootstrap"; + assets = [ (root + "/proto") ]; + }; + openshell-cli = { + dir = "openshell-cli"; + nativeCheckInputs = [ + pkgs.cacert + pkgs.git + ]; + assets = [ + (root + "/proto") + (root + "/providers") + (root + "/crates/openshell-prover/registry") + ]; + }; + openshell-server = { + dir = "openshell-server"; + assets = [ + (root + "/proto") + (root + "/providers") + (root + "/crates/openshell-prover/registry") + (root + "/crates/openshell-server/migrations") + (root + "/deploy/rpm/gateway.toml.default") + ]; + cargoTestExtraArgs = "--features test-support"; + }; + openshell-core = { + dir = "openshell-core"; + nativeBuildInputs = [ pkgs.protobuf ]; + assets = [ (root + "/proto") ]; + }; + openshell-driver-docker = { + dir = "openshell-driver-docker"; + assets = [ (root + "/proto") ]; + }; + openshell-sandbox = { + dir = "openshell-sandbox"; + nativeCheckInputs = [ + pkgs.bash + pkgs.coreutils + ]; + assets = [ + (root + "/proto") + (root + "/crates/openshell-supervisor-network/data") + (root + "/crates/openshell-supervisor-network/testdata") + (root + "/crates/openshell-supervisor-process/src/skills") + ]; + }; + openshell-driver-vm = { + dir = "openshell-driver-vm"; + assets = [ + (root + "/proto") + (root + "/crates/openshell-driver-vm/scripts") + ]; + }; + openshell-driver-kubernetes = { + dir = "openshell-driver-kubernetes"; + assets = [ (root + "/proto") ]; + }; + openshell-driver-podman = { + dir = "openshell-driver-podman"; + assets = [ (root + "/proto") ]; + }; + openshell-ocsf = { + dir = "openshell-ocsf"; + assets = [ (root + "/crates/openshell-ocsf/schemas") ]; + }; + openshell-policy = { + dir = "openshell-policy"; + assets = [ (root + "/proto") ]; + }; + openshell-prover = { + dir = "openshell-prover"; + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = [ pkgs.z3 ]; + env.LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + assets = [ + (root + "/crates/openshell-prover/registry") + (root + "/crates/openshell-prover/testdata") + ]; + }; + openshell-providers = { + dir = "openshell-providers"; + assets = [ + (root + "/proto") + (root + "/providers") + ]; + }; + openshell-router = { + dir = "openshell-router"; + assets = [ (root + "/proto") ]; + }; + openshell-server-macros = { + dir = "openshell-server-macros"; + }; + openshell-tui = { + dir = "openshell-tui"; + assets = [ + (root + "/proto") + (root + "/providers") + ]; + }; + openshell-vfio = { + dir = "openshell-vfio"; + }; +} diff --git a/nix/images.nix b/nix/images.nix new file mode 100644 index 000000000..c8dcbb732 --- /dev/null +++ b/nix/images.nix @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ + pkgs, + gateway, + supervisor, +}: +{ + openshell-gateway-image = pkgs.dockerTools.buildLayeredImage { + name = "openshell/gateway"; + tag = "nix"; + + contents = [ + gateway + pkgs.cacert + ]; + + extraCommands = '' + mkdir -p app usr/local/bin + cp --dereference ${gateway}/bin/openshell-gateway usr/local/bin/openshell-gateway + chmod 0555 usr/local/bin/openshell-gateway + ''; + + config = { + Entrypoint = [ "/usr/local/bin/openshell-gateway" ]; + Cmd = [ + "--bind-address" + "0.0.0.0" + "--port" + "8080" + ]; + Env = [ "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" ]; + ExposedPorts = { + "8080/tcp" = { }; + }; + User = "1000:1000"; + WorkingDir = "/app"; + }; + }; + + openshell-supervisor-image = pkgs.dockerTools.buildLayeredImage { + name = "openshell/supervisor"; + tag = "nix"; + + contents = [ supervisor ]; + + extraCommands = '' + cp --dereference ${supervisor}/bin/openshell-sandbox openshell-sandbox + chmod 0550 openshell-sandbox + ''; + + config = { + Entrypoint = [ "/openshell-sandbox" ]; + }; + }; +} diff --git a/nix/workspace.nix b/nix/workspace.nix new file mode 100644 index 000000000..dce48ceea --- /dev/null +++ b/nix/workspace.nix @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Crate-by-crate crane helpers for a Cargo workspace. +# +# Each crate is built from a minimal source, its own code plus that of its +# transitive workspace dependencies, and gets its own dependency cache. Crates +# outside that closure are reduced to their Cargo.toml so cargo can resolve +# the workspace without their source and without any Cargo.toml edits. +# Editing one crate never rebuilds an unrelated crate, and (because crane +# launders the source before building deps) never rebuilds any crate's +# dependency cache. +{ + lib, + pkgs, + craneLib, + # Workspace root: holds the virtual Cargo.toml, Cargo.lock and .cargo/. + root, + # Member directory, relative to root. + crateDir ? "crates", + # Crate metadata keyed by workspace crate directory. + crateSpecs ? { }, + # Version stamped onto every crate derivation. + version ? "0.0.0", +}: +let + cratesRoot = root + "/${crateDir}"; + + # Workspace dependency graph, derived from the Cargo.tomls + crateDirs = lib.attrNames (lib.filterAttrs (_: t: t == "directory") (builtins.readDir cratesRoot)); + + # Direct intra-workspace path-dependencies of a crate, as dir names. + directDeps = + dir: + let + manifest = builtins.fromTOML (builtins.readFile (cratesRoot + "/${dir}/Cargo.toml")); + in + lib.pipe (manifest.dependencies or { }) [ + lib.attrValues + (lib.filter (v: builtins.isAttrs v && v ? path)) + (map (v: baseNameOf v.path)) + (lib.filter (d: builtins.elem d crateDirs)) + ]; + + # Transitive closure of a crate within the workspace: its own dir plus every workspace dep. + closureOf = + dir: + map (e: e.key) ( + builtins.genericClosure { + startSet = [ { key = dir; } ]; + operator = e: map (key: { inherit key; }) (directDeps e.key); + } + ); + + specFor = dir: lib.attrByPath [ dir ] { } crateSpecs; + + closureList = closure: field: lib.concatLists (map (d: (specFor d).${field} or [ ]) closure); + + closureEnv = closure: lib.foldl' lib.recursiveUpdate { } (map (d: (specFor d).env or { }) closure); + + # Every member's Cargo.toml, cargo must see all of them to resolve the + # workspace even for crates whose source we leave out. + allManifests = map (d: cratesRoot + "/${d}/Cargo.toml") crateDirs; + + # Source tree carrying the real sources of the given crate dirs, plus every + # member's Cargo.toml and the given assets. + mkSrc = + { + dirs, + assets ? [ ], + }: + lib.fileset.toSource { + inherit root; + fileset = lib.fileset.unions ( + [ + (root + "/Cargo.toml") + (root + "/Cargo.lock") + (root + "/.cargo") + ] + ++ allManifests + ++ map (d: craneLib.fileset.commonCargoSources (cratesRoot + "/${d}")) dirs + ++ assets + ); + }; + + # Build one workspace crate (pname == dir) in three cached layers. Every layer + # uses the SAME `-p ` selection, so cargo's feature unification is + # identical across them and the compiled artifacts are reusable: + # 1. crates.io deps — buildDepsOnly; immune to first-party code. + # 2. workspace-dep libs — build `-p ` with the crate's OWN source + # stubbed (real path-deps), so its libs compile with + # the crate's real feature set and get cached. + # 3. the crate itself — reuses 1 + 2; only the crate's own code recompiles. + buildWorkspaceCrate = + { + dir, + assets ? [ ], + nativeBuildInputs ? [ ], + nativeCheckInputs ? [ ], + buildInputs ? [ ], + env ? { }, + cargoTestExtraArgs ? "", + }: + let + closure = closureOf dir; + workspaceDeps = lib.filter (d: d != dir) closure; + effectiveNativeBuildInputs = lib.unique ( + closureList closure "nativeBuildInputs" ++ nativeBuildInputs + ); + effectiveBuildInputs = lib.unique (closureList closure "buildInputs" ++ buildInputs); + effectiveEnv = lib.recursiveUpdate (closureEnv closure) env; + src = mkSrc { + dirs = closure; + inherit assets; + }; + common = { + pname = dir; + inherit version; + nativeBuildInputs = effectiveNativeBuildInputs; + buildInputs = effectiveBuildInputs; + env = effectiveEnv; + strictDeps = true; + # Build only, skip the cargo test/check phase for now. + doCheck = false; + cargoExtraArgs = "--locked -p ${dir}"; + }; + + cratesDeps = craneLib.buildDepsOnly (common // { src = mkSrc { dirs = [ ]; }; }); + + mkWorkspaceLibsSrc = + let + base = mkSrc { + dirs = workspaceDeps; + inherit assets; + }; + dummyCrate = craneLib.mkDummySrc { src = cratesRoot + "/${dir}"; }; + in + pkgs.runCommandLocal "source" { } '' + cp -r ${base} $out + chmod -R u+w $out + rm -rf "$out/${crateDir}/${dir}" + cp -r ${dummyCrate} "$out/${crateDir}/${dir}" + ''; + + workspaceLibs = + if workspaceDeps == [ ] then + cratesDeps + else + craneLib.buildPackage ( + common + // { + pname = "${dir}-workspace-libs"; + src = mkWorkspaceLibsSrc; + cargoArtifacts = cratesDeps; + doInstallCargoArtifacts = true; + postInstall = '' + cargo clean --release -p ${dir} + ''; + } + ); + + cratesTestDeps = craneLib.buildPackage ( + common + // { + pname = "${dir}-test-deps"; + src = mkWorkspaceLibsSrc; + inherit nativeCheckInputs; + cargoArtifacts = workspaceLibs; + cargoExtraArgs = "${common.cargoExtraArgs} --tests ${cargoTestExtraArgs}"; + doInstallCargoArtifacts = true; + } + ); + in + let + package = craneLib.buildPackage ( + common + // { + inherit src; + cargoArtifacts = workspaceLibs; + } + ); + + test = craneLib.cargoTest ( + common + // { + doCheck = true; + inherit src nativeCheckInputs cargoTestExtraArgs; + cargoArtifacts = cratesTestDeps; + } + ); + + clippy = craneLib.cargoClippy ( + common + // { + inherit src; + nativeBuildInputs = lib.unique (effectiveNativeBuildInputs ++ nativeCheckInputs); + cargoArtifacts = cratesTestDeps; + cargoClippyExtraArgs = "--all-targets -- -D warnings"; + } + ); + in + { + inherit package test clippy; + }; +in +{ + inherit buildWorkspaceCrate; +} diff --git a/scripts/update_license_headers.py b/scripts/update_license_headers.py index f56dbc293..4780b9049 100755 --- a/scripts/update_license_headers.py +++ b/scripts/update_license_headers.py @@ -43,6 +43,7 @@ ".yaml": "#", ".yml": "#", ".rego": "#", + ".nix": "#", } # Directories to skip entirely (relative to repo root).