diff --git a/.github/workflows/agentshield-loop.yml b/.github/workflows/agentshield-loop.yml index 970264e5..b655a3eb 100644 --- a/.github/workflows/agentshield-loop.yml +++ b/.github/workflows/agentshield-loop.yml @@ -23,11 +23,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: key: agentshield-loop-v0.8.7 @@ -53,7 +53,7 @@ jobs: - name: Upload loop state if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: agentshield-loop-state path: docs/loops/agentshield-scan/STATE.md diff --git a/.github/workflows/agentshield.yml b/.github/workflows/agentshield.yml index a2ddacb1..da0268b1 100644 --- a/.github/workflows/agentshield.yml +++ b/.github/workflows/agentshield.yml @@ -15,10 +15,10 @@ jobs: name: Security scan runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Run AgentShield - uses: aiconnai/agentshield@v0.8.7 + uses: aiconnai/agentshield@663cf4bf3c7a633c5caad398105264eb8680679d # v0.8.7 with: path: . fail-on: high diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3379360f..618a6688 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,28 +38,30 @@ permissions: # ============================================================================= # CI Policy (as of 2026-05) # -# Required to be green on every PR (cheap + deterministic): +# Required to be green on every PR: # - fmt # - clippy # - test (ubuntu-latest only) # - docs (includes ./scripts/generate-mcp-reference.sh --check) # -# These four jobs protect main. Contributors should run `make ci` or `just ci` -# locally before pushing. +# These four branch-protection contexts protect main. `Test (ubuntu-latest)` +# also depends transitively on the PR-visible `Security Gate`, which aggregates +# the supply-chain and security scans below. Contributors should run `make ci` +# or `just ci` locally before pushing. # -# Everything else (property-tests, golden-tests, full-feature clippy, backend -# smoke, benchmarks, cargo-deny, security-audit, release builds) is "extended". -# It runs on the nightly schedule, on manual dispatch, AND on every direct push -# to `main` as a post-merge safety net. It does NOT run on PRs, and it does NOT -# block: a push to `main` is already merged, so a red extended job is a signal -# to investigate, not a gate that holds anything back. +# Property tests, golden tests, full-feature Clippy, backend smoke, benchmarks, +# and release builds remain extended. They run on the nightly schedule, manual +# dispatch, and (where configured below) direct pushes to `main`; they do not +# block pull requests. Security and supply-chain jobs are the exception: they +# run on pull requests and feed the required aggregate gate. # # The heaviest long-tail jobs (`Full Feature Tests` which compiles # `--all-features` single-threaded, `Test (macos-latest)`, and `Coverage`) are # trimmed even further — schedule + manual dispatch only, NOT on push to `main` # — so normal `main` pushes never wait on them. Use `workflow_dispatch` to run -# them on demand. In all cases the four required jobs above are the gate; do not -# wait on any extended job before considering a `main` push landed. +# them on demand. In all cases the required jobs above plus the transitive +# Security Gate are the gate; do not wait on unrelated extended jobs before +# considering a `main` push landed. # # This keeps GitHub Actions as an enforceable gate without incurring high # per-PR costs (especially macOS minutes and long-running jobs). @@ -77,8 +79,8 @@ jobs: name: Format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 with: components: rustfmt - run: cargo fmt --all -- --check @@ -87,8 +89,8 @@ jobs: name: Clippy runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 with: components: clippy - name: Install protoc @@ -102,7 +104,7 @@ jobs: echo "No supported package manager found for protoc" >&2 exit 1 fi - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: shared-key: required-linux - run: | @@ -113,10 +115,10 @@ jobs: test: name: Test (ubuntu-latest) runs-on: ubuntu-latest - needs: [fmt, clippy] + needs: [fmt, clippy, security-gate] steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 with: targets: wasm32-unknown-unknown - name: Install protoc @@ -139,7 +141,7 @@ jobs: sudo apt-get update sudo apt-get install -y mold echo "RUSTFLAGS=-C link-arg=-fuse-ld=mold" >> "$GITHUB_ENV" - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: shared-key: required-linux - name: Test library and integration tests @@ -166,8 +168,8 @@ jobs: runs-on: macos-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 - name: Install protoc run: | if command -v apt-get >/dev/null 2>&1; then @@ -179,7 +181,7 @@ jobs: echo "No supported package manager found for protoc" >&2 exit 1 fi - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - name: macOS smoke test run: | source scripts/ci-features.env @@ -196,9 +198,9 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - run: cargo test --test property_tests -- --test-threads=1 env: PROPTEST_CASES: 500 @@ -209,9 +211,9 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - run: cargo test --test golden_tests full-feature-clippy: @@ -219,8 +221,8 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 with: components: clippy - name: Install protoc @@ -238,7 +240,7 @@ jobs: run: | sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/share/boost || true docker system prune -af || true - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: shared-key: extended-linux - run: cargo clippy --all-targets --all-features -- -D warnings @@ -248,8 +250,8 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 - name: Install protoc run: | if command -v apt-get >/dev/null 2>&1; then @@ -270,7 +272,7 @@ jobs: sudo apt-get update sudo apt-get install -y mold echo "RUSTFLAGS=-C link-arg=-fuse-ld=mold" >> "$GITHUB_ENV" - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: shared-key: extended-linux - run: | @@ -282,8 +284,8 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 - name: Free Linux runner disk space run: | sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/share/boost || true @@ -293,7 +295,7 @@ jobs: sudo apt-get update sudo apt-get install -y mold echo "RUSTFLAGS=-C link-arg=-fuse-ld=mold" >> "$GITHUB_ENV" - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: shared-key: extended-linux - name: Test local embedding backend @@ -308,16 +310,14 @@ jobs: audit: name: Security Audit runs-on: ubuntu-latest - # Also runs on PRs as an ADVISORY signal (surfaces advisories early). It is - # NOT a required status check until its baseline is clean and stable — see - # github-harness-protection Todo 6/8. Keeping it off the required list means - # a red advisory does not block merge. + # Runs on PRs and feeds the aggregate Security Gate. The live branch + # protection context may also require this job directly. if: github.event_name == 'pull_request' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 - name: Install cargo-audit - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2 with: tool: cargo-audit - name: Run cargo audit @@ -326,16 +326,104 @@ jobs: deny: name: Cargo Deny runs-on: ubuntu-latest - # Also runs on PRs as an ADVISORY signal (license/ban/source/advisory drift). - # NOT a required status check until baseline is clean and stable — see - # github-harness-protection Todo 6/8. Off the required list = does not block. + # Runs on PRs and feeds the aggregate Security Gate. The live branch + # protection context may also require this job directly. if: github.event_name == 'pull_request' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v5 - - uses: EmbarkStudios/cargo-deny-action@v2 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: EmbarkStudios/cargo-deny-action@8f84122a46a358a27cb0625d85ad60ab436a1b87 # v2 with: command: check advisories bans licenses sources + security-exceptions: + name: Security exception policy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Validate governed advisory exceptions + run: python3 scripts/check-security-exceptions.py --config docs/security/advisory-exceptions.toml --audit-config .cargo/audit.toml --deny-config deny.toml + + codeql-security: + name: CodeQL security + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: github/codeql-action/init@641a925cfafe92d0fdf8b239ba4053e3f8d99d6d # v3 + with: + languages: rust + config-file: ./.github/codeql/codeql-config.yml + - uses: github/codeql-action/analyze@641a925cfafe92d0fdf8b239ba4053e3f8d99d6d # v3 + with: + category: /language:rust + + semgrep-security: + name: Semgrep security + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Run Semgrep + run: | + docker run --rm \ + -v "${{ github.workspace }}:/src" \ + semgrep/semgrep:1.169.0 \ + semgrep --config p/ci --error /src + + gitleaks-security: + name: Secret scan security + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + fetch-depth: 0 + - name: Run Gitleaks + run: | + docker run --rm \ + -v "${{ github.workspace }}:/src" \ + zricethezav/gitleaks:v8.23.3 \ + detect --source /src --no-git --verbose --redact --exit-code 1 + + agentshield-security: + name: AgentShield security + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: aiconnai/agentshield@663cf4bf3c7a633c5caad398105264eb8680679d # v0.8.7 + with: + path: . + fail-on: high + ignore-tests: true + format: sarif + upload-sarif: false + version: v0.8.7 + + security-gate: + name: Security Gate + runs-on: ubuntu-latest + if: always() + needs: + - audit + - deny + - security-exceptions + - codeql-security + - semgrep-security + - gitleaks-security + - agentshield-security + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - name: Require every security constituent + env: + RESULTS_JSON: ${{ toJSON(needs) }} + run: >- + python3 scripts/check-security-gate.py + --matrix tests/fixtures/security_gate_matrix.json + --workflow .github/workflows/ci.yml + --results-json "$RESULTS_JSON" + --event "${{ github.event_name }}" + # ============================================================================= # Coverage (with threshold) # ============================================================================= @@ -345,8 +433,8 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 with: components: llvm-tools-preview - name: Install protoc @@ -364,12 +452,12 @@ jobs: run: | sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/share/boost || true docker system prune -af || true - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: cache-targets: false - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov + uses: taiki-e/install-action@9458fed45835344deb5e69e06e831be047c96abf # cargo-llvm-cov - name: Generate coverage run: | @@ -392,7 +480,7 @@ jobs: cargo llvm-cov report --fail-under-lines 55 - name: Upload to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4 with: files: lcov.info fail_ci_if_error: false @@ -408,9 +496,9 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - run: cargo bench --no-run bench: @@ -421,15 +509,15 @@ jobs: contents: write pull-requests: write steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - name: Run benchmarks (Criterion JSON output) run: cargo bench -- --output-format bencher | tee bench-output.txt - name: Store benchmark results - uses: benchmark-action/github-action-benchmark@v1 + uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1 # Tolerate missing gh-pages branch on first run / forks. The raw # criterion artifacts still upload via the next step, so PR authors # can inspect benchmark output even before history is established. @@ -452,7 +540,7 @@ jobs: benchmark-data-dir-path: dev/bench - name: Upload raw Criterion reports - uses: actions/upload-artifact@v7.0.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: benchmark-results @@ -466,8 +554,8 @@ jobs: name: Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 - name: Install protoc run: | if command -v apt-get >/dev/null 2>&1; then @@ -479,7 +567,7 @@ jobs: echo "No supported package manager found for protoc" >&2 exit 1 fi - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: shared-key: required-linux - run: ./scripts/generate-mcp-reference.sh --check @@ -509,17 +597,17 @@ jobs: - os: macos-latest target: aarch64-apple-darwin steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable 2026-07-12 with: targets: ${{ matrix.target }} - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - name: Build release run: cargo build --release --target ${{ matrix.target }} --features pdf --bin engram-server --bin engram-cli --bin engram-pdf-worker - name: Upload artifacts - uses: actions/upload-artifact@v7.0.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: engram-${{ matrix.target }} path: | @@ -527,7 +615,7 @@ jobs: target/${{ matrix.target }}/release/engram-cli - name: Upload PDF worker artifact if: contains(matrix.target, '-linux-') - uses: actions/upload-artifact@v7.0.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: engram-pdf-worker-${{ matrix.target }} path: | diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3792cdd8..4d36b0b3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -24,15 +24,15 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@641a925cfafe92d0fdf8b239ba4053e3f8d99d6d # v3 with: languages: ${{ matrix.language }} config-file: ./.github/codeql/codeql-config.yml - name: Run CodeQL analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@641a925cfafe92d0fdf8b239ba4053e3f8d99d6d # v3 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index 99ac6ef6..75f4840e 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -15,7 +15,7 @@ jobs: name: Secret scan runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 0 @@ -35,7 +35,7 @@ jobs: - name: Upload SARIF to GitHub Security if: always() - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@1ad29ea4a422cce9a242a9fae469541dcd08addc # v4 with: sarif_file: gitleaks.sarif category: gitleaks diff --git a/.github/workflows/harness-contract.yml b/.github/workflows/harness-contract.yml index e65ec9b4..4c849ea2 100644 --- a/.github/workflows/harness-contract.yml +++ b/.github/workflows/harness-contract.yml @@ -35,7 +35,7 @@ jobs: name: Harness Contract runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - name: Harness bootstrap sanity (read-only) run: bash docs/harness/bin/bootstrap.sh @@ -59,7 +59,7 @@ jobs: runs-on: ubuntu-latest continue-on-error: true steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - name: Harness doctor (advisory, non-blocking) continue-on-error: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 88909147..09571436 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,9 +24,9 @@ jobs: name: Fuzz Testing runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@nightly - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@efcb852328a9f50117170cc43094fb6f09eaf1ae # nightly + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - name: Install cargo-fuzz run: cargo install cargo-fuzz @@ -49,9 +49,9 @@ jobs: # Only run on Sundays if: github.event.schedule == '0 3 * * 0' || github.event_name == 'workflow_dispatch' steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - name: Install cargo-mutants run: cargo install cargo-mutants @@ -61,7 +61,7 @@ jobs: continue-on-error: true - name: Upload mutation report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: mutation-report path: mutants.out/ @@ -74,9 +74,9 @@ jobs: name: Extended Property Tests runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - name: Run extended property tests run: cargo test --test property_tests -- --test-threads=1 @@ -91,11 +91,11 @@ jobs: name: Miri (Undefined Behavior Check) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@nightly + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@efcb852328a9f50117170cc43094fb6f09eaf1ae # nightly with: components: miri - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - name: Run Miri on critical modules run: | @@ -110,8 +110,8 @@ jobs: name: Check Outdated Dependencies runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable - name: Install cargo-outdated run: cargo install cargo-outdated @@ -130,15 +130,15 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v5 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 - name: Run all benchmarks run: cargo bench -- --output-format bencher | tee bench-output.txt - name: Track benchmark results - uses: benchmark-action/github-action-benchmark@v1 + uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1 with: name: Engram Performance (Nightly) tool: cargo @@ -151,7 +151,7 @@ jobs: benchmark-data-dir-path: dev/bench-nightly - name: Upload raw Criterion reports - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() with: name: nightly-benchmarks diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 20119d12..8a872836 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -87,15 +87,15 @@ jobs: target: aarch64-unknown-linux-gnu steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: ref: ${{ env.RELEASE_VERSION }} - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable with: targets: ${{ matrix.target }} - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: key: release-${{ matrix.target }} @@ -123,7 +123,7 @@ jobs: shasum -a 256 "${ARCHIVE}" > "${ARCHIVE}.sha256" - name: Upload artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ matrix.target }} path: | @@ -135,12 +135,12 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: ref: ${{ env.RELEASE_VERSION }} - name: Download all artifacts - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: path: artifacts @@ -157,7 +157,7 @@ jobs: cat release/RELEASE_NOTES.md - name: Create GitHub Release - uses: softprops/action-gh-release@v3 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3 with: tag_name: ${{ env.RELEASE_VERSION }} draft: false @@ -187,20 +187,20 @@ jobs: - name: Download SHA256 files if: steps.homebrew-token.outputs.available == 'true' - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: path: artifacts - name: Checkout Engram release source if: steps.homebrew-token.outputs.available == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: path: engram-source ref: ${{ env.RELEASE_VERSION }} - name: Checkout homebrew-engram if: steps.homebrew-token.outputs.available == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: repository: aiconnai/homebrew-engram token: ${{ env.HOMEBREW_TAP_TOKEN }} diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index d04c0847..931494b9 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -12,29 +12,21 @@ permissions: jobs: scan: - name: Semgrep (optional) + name: Semgrep runs-on: ubuntu-latest - continue-on-error: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Run Semgrep run: | - set +e docker run --rm \ -v "${{ github.workspace }}:/src" \ - returntocorp/semgrep:latest \ - semgrep --config p/ci --sarif --output /src/semgrep.sarif /src - EXIT_CODE=$? - set -e - - if [ "$EXIT_CODE" -gt 1 ]; then - exit $EXIT_CODE - fi + semgrep/semgrep:1.169.0 \ + semgrep --config p/ci --error --sarif --output /src/semgrep.sarif /src - name: Upload SARIF result if: always() - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@1ad29ea4a422cce9a242a9fae469541dcd08addc # v4 with: sarif_file: semgrep.sarif category: semgrep diff --git a/docs/MCP_AUTH.md b/docs/MCP_AUTH.md index 8c71c09b..7a0998c9 100644 --- a/docs/MCP_AUTH.md +++ b/docs/MCP_AUTH.md @@ -62,8 +62,20 @@ Engram can enforce a token-bucket rate limit for MCP HTTP requests: - `--http-rate-limit-burst` / `ENGRAM_HTTP_RATE_LIMIT_BURST` (default: `240`) - `--http-rate-limit-key` / `ENGRAM_HTTP_RATE_LIMIT_KEY` (optional identity header) -When the key is unset, bucket keys are derived from `x-forwarded-for`, -then `x-real-ip`, then `ip:unknown`. +When the key is unset, bucket keys use the TCP socket peer address. Forwarded +identity is accepted only when that peer matches a CIDR in +`ENGRAM_HTTP_TRUSTED_PROXIES` (comma-separated IPv4/IPv6 CIDRs). Engram then +normalizes `X-Forwarded-For` from right to left across trusted hops. Malformed, +empty, or overlong chains fall back to the socket peer. `X-Real-IP` is never +trusted implicitly. + +Example for a loopback reverse proxy and a private proxy tier: + +```bash +ENGRAM_HTTP_TRUSTED_PROXIES="127.0.0.0/8,10.0.0.0/8" +``` + +Leave the variable unset to disable trusted-proxy mode. When a key header is set, its value is used as the bucket identity key. diff --git a/docs/harness/.sensors-last b/docs/harness/.sensors-last index 47b2eb89..9bab43d4 100644 --- a/docs/harness/.sensors-last +++ b/docs/harness/.sensors-last @@ -1,7 +1,7 @@ status=pass ci_status=pass doctor_status=pass -mode=quick -timestamp=2026-07-12T07:40:59Z -duration_sec=30 -ci=cargo fmt + cargo check + pr-title-policy + harness doctor +mode=full +timestamp=2026-07-12T17:04:23Z +duration_sec=40 +ci=fmt + clippy + test_lib + test_integration + test_integration_watch + wasm_target + wasm_all_targets + wasm_wasm_target + doc + ref_check + pr-title-policy + harness doctor diff --git a/docs/harness/.sensors-log b/docs/harness/.sensors-log index a2e3397b..332dd7d0 100644 --- a/docs/harness/.sensors-log +++ b/docs/harness/.sensors-log @@ -42,3 +42,4 @@ {"schema_version":"sensors-log-v1","timestamp":"2026-07-09T20:18:57Z","tool":"sensors","mode":"full","status":"pass","duration_sec":30,"ci_status":"pass","doctor_status":"pass","ci_command":"fmt + clippy + test_lib + test_integration + test_integration_watch + wasm_target + wasm_all_targets + wasm_wasm_target + doc + ref_check + pr-title-policy + harness doctor","ci_steps":{"fmt":"pass","clippy":"pass","test_lib":"pass","test_integration":"pass","test_integration_watch":"pass","wasm_target":"pass","wasm_all_targets":"pass","wasm_wasm_target":"pass","doc":"pass","ref_check":"pass"},"exclusion":null,"artifacts":[{"path":"docs/harness/.sensors-last","kind":"sensors_last","format":"key_value"}]} {"schema_version":"sensors-log-v1","timestamp":"2026-07-10T00:27:38Z","tool":"sensors","mode":"full","status":"pass","duration_sec":89,"ci_status":"pass","doctor_status":"pass","ci_command":"fmt + clippy + test_lib + test_integration + test_integration_watch + wasm_target + wasm_all_targets + wasm_wasm_target + doc + ref_check + pr-title-policy + harness doctor","ci_steps":{"fmt":"pass","clippy":"pass","test_lib":"pass","test_integration":"pass","test_integration_watch":"pass","wasm_target":"pass","wasm_all_targets":"pass","wasm_wasm_target":"pass","doc":"pass","ref_check":"pass"},"exclusion":null,"artifacts":[{"path":"docs/harness/.sensors-last","kind":"sensors_last","format":"key_value"}]} {"schema_version":"sensors-log-v1","timestamp":"2026-07-12T07:40:59Z","tool":"sensors","mode":"quick","status":"pass","duration_sec":30,"ci_status":"pass","doctor_status":"pass","ci_command":"cargo fmt + cargo check + pr-title-policy + harness doctor","ci_steps":{},"exclusion":null,"artifacts":[{"path":"docs/harness/.sensors-last","kind":"sensors_last","format":"key_value"}]} +{"schema_version":"sensors-log-v1","timestamp":"2026-07-12T17:04:23Z","tool":"sensors","mode":"full","status":"pass","duration_sec":40,"ci_status":"pass","doctor_status":"pass","ci_command":"fmt + clippy + test_lib + test_integration + test_integration_watch + wasm_target + wasm_all_targets + wasm_wasm_target + doc + ref_check + pr-title-policy + harness doctor","ci_steps":{"fmt":"pass","clippy":"pass","test_lib":"pass","test_integration":"pass","test_integration_watch":"pass","wasm_target":"pass","wasm_all_targets":"pass","wasm_wasm_target":"pass","doc":"pass","ref_check":"pass"},"exclusion":null,"artifacts":[{"path":"docs/harness/.sensors-last","kind":"sensors_last","format":"key_value"}]} diff --git a/docs/harness/GATES.md b/docs/harness/GATES.md index 6347ba45..e8eeaa3a 100644 --- a/docs/harness/GATES.md +++ b/docs/harness/GATES.md @@ -85,6 +85,34 @@ Saídas JSON opt-in para scripts do harness devem seguir vocabulário de status estável, exit code preservado e nenhum segredo ou dump de ambiente. O output humano continua sendo o default. +### Required aggregate security gate + +The PR-visible `Security Gate` job in `.github/workflows/ci.yml` aggregates +Cargo Audit, Cargo Deny, governed exception-policy validation, CodeQL, +Semgrep, Gitleaks, and AgentShield. Every constituent is release-blocking on a +pull request: a failed or unexpectedly skipped constituent makes the aggregate +red. Event-policy skips are neutral only when explicitly listed in +`tests/fixtures/security_gate_matrix.json`; the current pull-request policy has +no such skip. + +Branch protection itself is not changed by repository automation. Instead, the +already-required `Test (ubuntu-latest)` job depends on `security-gate`, so a +security failure blocks that required context transitively. Verify the live, +read-only chain with: + +```bash +gh api repos/aiconnai/engram/branches/main/protection/required_status_checks \ + > .omo/evidence/task-18-required-contexts.json +python3 scripts/check-security-gate.py \ + --matrix tests/fixtures/security_gate_matrix.json \ + --required-contexts .omo/evidence/task-18-required-contexts.json \ + --workflow .github/workflows/ci.yml +``` + +The checker also exposes `--self-test-failure` and `--self-test-unrequired` to +prove that a constituent failure or removal of the required-context dependency +fails closed. + ### PR Title Policy Wrapper: `bash docs/harness/bin/pr-title-policy.sh` diff --git a/docs/harness/README.md b/docs/harness/README.md index 2ea9bc39..71fcd311 100644 --- a/docs/harness/README.md +++ b/docs/harness/README.md @@ -85,10 +85,14 @@ No GitHub, o merge em `main` exige estes **required status checks**: Commits). O `doctor.sh` **não** faz parte deste gate required — ele permanece local/advisory. -`Security Audit` e `Cargo Deny` rodam nos PRs como sinais **advisory** (ainda não -bloqueiam merge; ver baseline em [`GATES.md`](./GATES.md)). Code review automático -(Copilot / terceiros) é **sinal extra, não autoritativo**: por si só não bloqueia -o merge enquanto não expuser um status check confiável. +`Test (ubuntu-latest)` depende do job PR-visible `Security Gate`, que agrega +Cargo Audit, Cargo Deny, a política de exceções, CodeQL, Semgrep, Gitleaks e +AgentShield. Assim, qualquer falha constituinte bloqueia transitivamente um +contexto já required sem uma escrita automatizada em branch protection. + +Code review automático (Copilot / terceiros) é **sinal extra, não +autoritativo**: por si só não bloqueia o merge enquanto não expuser um status +check confiável. ## Estrutura diff --git a/docs/harness/canvas/2026-07-12-security-aggregate-gate.md b/docs/harness/canvas/2026-07-12-security-aggregate-gate.md new file mode 100644 index 00000000..c0a2007b --- /dev/null +++ b/docs/harness/canvas/2026-07-12-security-aggregate-gate.md @@ -0,0 +1,84 @@ +# Review Canvas — Required Aggregate Security Gate + +| Field | Value | +|-------|-------| +| Task | `engram-10-of-10-todo-18` | +| Date | `2026-07-12` | +| Scope | GitHub Actions security jobs, aggregate contract checker, fixture, and gate documentation | +| Out of scope | Branch-protection writes, advisory-policy changes, product code, publication | + +## Problem + +Security and supply-chain scans were split across advisory or independent +workflow signals. A failure could therefore remain disconnected from the +already-required branch-protection contexts, and there was no deterministic +proof of the dependency chain. + +## Approaches Considered + +- Write branch protection directly: rejected because repository automation is + not authorized to mutate that external security boundary. +- Treat independent workflow statuses as the aggregate: rejected because + GitHub Actions cannot express cross-workflow `needs` dependencies. +- Run PR-visible constituent jobs in CI, aggregate them, and make the existing + required test context depend on the aggregate: selected because it fails + closed without an external write and remains statically verifiable. + +## Failure Semantics + +`security-gate` uses `if: always()` so a failed prerequisite cannot silently +skip the aggregate. It accepts only successful constituents on current PR +events. The matrix checker proves every constituent failure is red, an +explicit event-policy skip can be neutral, and removal of the required-context +dependency is rejected. + +## Hot-Path / Critical-Path Complexity + +The required Ubuntu test now waits for seven independent security constituents +and their small aggregate job. Those constituents run in parallel, so the PR +critical path grows by the slowest scan (normally CodeQL), not by the sum of all +seven jobs. This intentionally trades additional Actions time and merge latency +for a single fail-closed security boundary. The aggregate itself only checks a +bounded seven-entry JSON object and does not compile or scan the repository. + +The standalone scanner workflows remain available for GitHub Security/SARIF +visibility, while the CI copies are what create a deterministic `needs` graph. +Version pins and the contract checker keep those duplicated definitions +reviewable; future consolidation must preserve both SARIF visibility and the +required transitive chain. + +## Edge Cases + +- **Cancelled, timed-out, or missing constituent:** GitHub reports a non-success + result (or omits it). The checker treats every state other than `success` or + an explicitly event-allowed `skipped` as failure, so `Security Gate` and the + dependent required test remain red/skipped rather than passing silently. +- **Fork pull request has downgraded token permissions:** if CodeQL or SARIF + upload cannot run with the fork token, that constituent fails and the + aggregate blocks merge. It never falls back to a green advisory result or + exposes a repository secret. +- **Workflow event legitimately excludes a future constituent:** a skip is + neutral only after that exact job/event pairing is recorded in the versioned + matrix. Current PR, push, schedule, and dispatch policies allow no skips. +- **Branch-protection context is renamed or removed:** the read-only live + context receipt no longer matches a job that reaches `security-gate`, and + `check-security-gate.py` rejects the chain. + +## Breakage-Risk Table + +| Risk | Impact | Mitigation | +|------|--------|------------| +| Scan failure silently skips aggregate | Required test might still run | Aggregate uses `if: always()` and inspects every `needs` result | +| Aggregate is not merge-blocking | Red security result remains advisory | Required `Test (ubuntu-latest)` transitively needs `security-gate`; checker validates live contexts read-only | +| Mutable action tag changes executed code | Supply-chain drift | Third-party actions in owned workflows are pinned to immutable commits with version comments | +| Legitimate event exclusion is treated as green accidentally | False success | Skips require explicit matrix policy; current PR jobs have no skip allowance | +| Duplicate standalone scans drift | Conflicting signals | Standalone workflows use the same pinned scanner/action versions as aggregate jobs | + +## Reviewer Checklist + +- Confirm all seven constituents are direct dependencies of `security-gate`. +- Confirm `test` depends on `security-gate` and its live name remains required. +- Confirm constituent failures and unexpected skips fail closed. +- Confirm no branch-protection or publication write occurs. +- Confirm the checker negative self-tests fail for a constituent failure and an + unrequired aggregate. diff --git a/docs/harness/progress.md b/docs/harness/progress.md index df3a9cae..decbd549 100644 --- a/docs/harness/progress.md +++ b/docs/harness/progress.md @@ -7,9 +7,9 @@ | Active task | `engram-10-of-10-live-state — make harness live state truthful and self-checking` | | Active plan | `docs/harness/progress/2026-06-27-harness-live-state-closeout.md` | | Last review | `2026-07-10 — pass: docs/harness/reviews/2026-07-10-engram-10-of-10-live-state-v4-post.md` | -| Last sensors | `2026-07-12 — status=pass (mode=quick; timestamp 2026-07-12T07:40:59Z)` | -| Last commit | `2272d6c8796945068d019a8116ae6bc569f361cf` | -| Last live-state check | `2026-07-10 — status=pass (rtk bash docs/harness/bin/check-live-state.sh --progress docs/harness/progress.md)` | +| Last sensors | `2026-07-12 — status=pass (mode=full; timestamp 2026-07-12T17:04:23Z)` | +| Last commit | `962428a` | +| Last live-state check | `2026-07-13 — status=pass (rtk bash docs/harness/bin/check-live-state.sh --progress docs/harness/progress.md)` | > Sumário curto do trabalho ativo. Logs detalhados em `progress/`. @@ -52,6 +52,39 @@ - Exact CI Clippy, clean-tree Gitleaks, focused storage/cloud/WS/listener tests, formatting, doctor, diff-check, and independent review passed locally. +## Engram 10/10 Wave 3 — execution started + +- PR #189 merged as `81be152c713230c082901899e6880579fcedabb3`; all + required checks were green before merge. +- Todos 17–21 are the next dependency-ready lanes and are being executed on + isolated branches/worktrees for later integration and independent review. +- Atomic descriptor-bound SQLite opening remains a separate deferred follow-up; + Wave 3 does not claim or silently absorb that work. +- No tag, release, deployment, registry publication, or channel approval is in + scope for this wave. + +### Wave 3 integration result + +- Todos 17–21 are integrated on `feat/engram-10-of-10-wave3`. +- HTTP trusted-proxy identity is explicit and bounded; the aggregate security + gate, canonical real-binary journey, frozen retrieval corpus, and example + smoke suite are executable and fail closed on their negative fixtures. +- `make ci` and the full harness sensors passed on the integrated branch with + no exclusions. +- Independent review found that the first canonical journey used separate + transport databases. Commit `4a70aa0` corrected the harness so stdio and + authenticated HTTP share one caller-owned isolated database; HTTP asserts + the record previously updated through stdio, and cleanup occurs only after + both transport lifecycles finish. +- PR #191 CI exposed a removed Semgrep image tag in both the standalone and + aggregate security jobs. Both workflows now use the verified official + `semgrep/semgrep:1.169.0` image; scanner policy and failure behavior are + unchanged. +- With the scanner restored, Semgrep identified 36 mutable action references + in four existing workflows. Those references are now pinned to resolved + commit SHAs while retaining version comments; the same `p/ci --error` scan + reports zero findings without ignores or exclusions. + - Tasks 11–16 landed as `bbd49fc` (HTTP fail-closed), `fc37fff` (gRPC security), `73f4959` (WebSocket authentication), `bc05e81` (durable cloud key identity), `815d1af` (SQLite permissions), and `ce292ac` (bounded PDF diff --git a/docs/harness/progress/2026-06-27-harness-live-state-closeout.md b/docs/harness/progress/2026-06-27-harness-live-state-closeout.md index 0c76fa48..aaae1798 100644 --- a/docs/harness/progress/2026-06-27-harness-live-state-closeout.md +++ b/docs/harness/progress/2026-06-27-harness-live-state-closeout.md @@ -234,3 +234,43 @@ evidence file under `.omo/evidence/`. - Quoted the complete expression without changing its runtime behavior. - Ruby YAML parse, harness doctor, fallback title policy, and quick sensors all passed. Independent Sonnet review returned `REVIEW_VERDICT: PASS`. + +## 2026-07-12 — Wave 3 execution start + +- Verified PR #189 merged at + `81be152c713230c082901899e6880579fcedabb3` with required checks green. +- Refreshed the canonical live-state commit and quick-sensor timestamp. +- Started dependency-ready Todos 17–21 in isolated worktrees for later + integration and independent review. +- Kept atomic descriptor-bound SQLite opening explicitly deferred and kept all + publication channels outside this wave. + +## 2026-07-12 — Wave 3 integration + +- Integrated Todos 17–21 after isolated focused verification. +- Added bounded trusted-proxy identity, the required aggregate security gate, + canonical stdio/HTTP real-binary journey, frozen retrieval-quality corpus, + and deterministic public-example smoke coverage. +- `make ci` and full sensors passed without exclusions on the integrated + branch. Independent post-review was requested against `origin/main..HEAD`. + +## 2026-07-13 — Wave 3 shared-state review remediation + +- Independent review proved the initial canonical journey created separate + temporary databases for stdio and HTTP. +- Commit `4a70aa0` moved database ownership to the caller, runs both transports + sequentially against the same isolated database, and verifies through HTTP + the record updated through stdio before final cleanup. +- Focused canonical tests and Clippy passed; live-state metadata was refreshed + before requesting the next immutable review. + +## 2026-07-13 — PR #191 Semgrep image remediation + +- Both Semgrep jobs failed before scanning because Docker Hub no longer served + `returntocorp/semgrep:1.129.0`. +- Updated the standalone and aggregate jobs to the verified official immutable + tag `semgrep/semgrep:1.169.0`, without adding ignores or weakening `--error`. +- The restored scanner then surfaced 36 pre-existing mutable GitHub Action + references across nightly, release, AgentShield loop, and harness-contract + workflows. Pinned each reference to its resolved commit SHA with a version + comment. The exact container scan now reports zero findings. diff --git a/examples/README.md b/examples/README.md index 5d445faf..59831054 100644 --- a/examples/README.md +++ b/examples/README.md @@ -3,6 +3,29 @@ These examples show how Engram fits into common agent ecosystems without overstating support. +## Deterministic smoke test + +From a clean checkout, run: + +```bash +bash scripts/test-examples.sh +``` + +The aggregate test uses a disposable database and a loopback-only real Engram +HTTP server. It never contacts an external service and requires no API keys. +The ecosystem-specific Python packages are intentionally not installed: their +dependency-free Engram adapters are syntax checked and exercised directly. + +| Family | Classification | Smoke coverage | +| --- | --- | --- | +| Rust library demos | Runnable | All Cargo examples compile | +| Claude MCP | Runnable | Real `memory_create` and `memory_search` HTTP round trip | +| Cursor MCP | Illustrative configuration | JSON block and stdio arguments validate | +| CrewAI memory | Illustrative adapter sketch | SDK adapter imports/classes validate | +| OpenAI Agents SDK | Runnable integration pattern | Dry run plus real Engram adapter round trip | +| FastMCP server | Runnable integration pattern | Dry run plus real Engram adapter round trip | +| LangGraph tool | Runnable integration pattern | Dry run plus real Engram adapter round trip | + ## Native Examples - [Claude MCP](claude-mcp/) - configure Engram as a Claude Code MCP memory server. diff --git a/scripts/check-security-gate.py b/scripts/check-security-gate.py new file mode 100755 index 00000000..c843f2dd --- /dev/null +++ b/scripts/check-security-gate.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Validate the aggregate security gate and its required-context chain.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + + +class CheckError(RuntimeError): + """A security-gate contract violation.""" + + +def load_json(path: Path) -> dict: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise CheckError(f"could not read JSON {path}: {exc}") from exc + if not isinstance(value, dict): + raise CheckError(f"{path}: root must be an object") + return value + + +def parse_jobs(path: Path) -> dict[str, dict[str, object]]: + """Parse the small job/name/needs subset used by the CI contract.""" + + try: + lines = path.read_text().splitlines() + except OSError as exc: + raise CheckError(f"could not read workflow {path}: {exc}") from exc + jobs: dict[str, dict[str, object]] = {} + in_jobs = False + current: str | None = None + collecting_needs = False + for line in lines: + if line == "jobs:": + in_jobs = True + continue + if not in_jobs: + continue + match = re.match(r"^ ([A-Za-z0-9_-]+):\s*$", line) + if match: + current = match.group(1) + jobs[current] = {"name": current, "needs": []} + collecting_needs = False + continue + if current is None: + continue + name = re.match(r"^ name:\s*(.+?)\s*$", line) + if name: + jobs[current]["name"] = name.group(1).strip("'\"") + continue + inline = re.match(r"^ needs:\s*\[(.*?)\]\s*$", line) + if inline: + jobs[current]["needs"] = [ + item.strip().strip("'\"") + for item in inline.group(1).split(",") + if item.strip() + ] + collecting_needs = False + continue + if re.match(r"^ needs:\s*$", line): + jobs[current]["needs"] = [] + collecting_needs = True + continue + dependency = re.match(r"^ -\s+([A-Za-z0-9_-]+)\s*$", line) + if collecting_needs and dependency: + needs = jobs[current]["needs"] + assert isinstance(needs, list) + needs.append(dependency.group(1)) + continue + if collecting_needs and line.strip() and not line.startswith(" "): + collecting_needs = False + return jobs + + +def evaluate(results: dict[str, str], constituents: list[str], allowed: set[str]) -> bool: + for job in constituents: + result = results.get(job, "missing") + if result == "success": + continue + if result == "skipped" and job in allowed: + continue + return False + return True + + +def validate_matrix(matrix: dict) -> None: + constituents = matrix.get("constituents") + scenarios = matrix.get("scenarios") + if not isinstance(constituents, list) or not constituents: + raise CheckError("matrix needs a non-empty constituents list") + if not all(isinstance(item, str) and item for item in constituents): + raise CheckError("matrix constituent IDs must be non-empty strings") + if not isinstance(scenarios, list) or not scenarios: + raise CheckError("matrix needs scenarios") + for scenario in scenarios: + results = scenario.get("results", {}) + allowed = set(scenario.get("allowed_skips", [])) + expected = scenario.get("expected") == "success" + if evaluate(results, constituents, allowed) != expected: + raise CheckError(f"matrix scenario disagrees with gate policy: {scenario.get('name')}") + baseline = {job: "success" for job in constituents} + if not evaluate(baseline, constituents, set()): + raise CheckError("all-success matrix must pass") + for job in constituents: + failed = dict(baseline) + failed[job] = "failure" + if evaluate(failed, constituents, set()): + raise CheckError(f"constituent failure did not fail closed: {job}") + + +def required_context_names(payload: dict) -> set[str]: + names = {item for item in payload.get("contexts", []) if isinstance(item, str)} + for item in payload.get("checks", []): + if isinstance(item, dict) and isinstance(item.get("context"), str): + names.add(item["context"]) + return names + + +def has_path(jobs: dict[str, dict[str, object]], start: str, target: str) -> bool: + pending = [start] + seen: set[str] = set() + while pending: + job = pending.pop() + if job == target: + return True + if job in seen: + continue + seen.add(job) + details = jobs.get(job, {}) + dependencies = details.get("needs", []) + if isinstance(dependencies, list): + pending.extend(item for item in dependencies if isinstance(item, str)) + return False + + +def validate_workflow( + matrix: dict, jobs: dict[str, dict[str, object]], workflow_text: str +) -> None: + aggregate = matrix.get("aggregate_job") + if aggregate not in jobs: + raise CheckError(f"workflow missing aggregate job {aggregate}") + needs = jobs[aggregate].get("needs", []) + missing = set(matrix["constituents"]) - set(needs if isinstance(needs, list) else []) + if missing: + raise CheckError(f"aggregate does not depend on: {', '.join(sorted(missing))}") + required_tokens = ( + "if: always()", + "scripts/check-security-gate.py", + "--results-json", + "toJSON(needs)", + ) + absent = [token for token in required_tokens if token not in workflow_text] + if absent: + raise CheckError(f"aggregate runtime enforcement missing: {', '.join(absent)}") + + +def validate_required_chain(matrix: dict, jobs: dict[str, dict[str, object]], payload: dict) -> None: + required = required_context_names(payload) + aggregate = str(matrix["aggregate_job"]) + candidates = [job for job, details in jobs.items() if details.get("name") in required] + if not any(has_path(jobs, job, aggregate) for job in candidates): + raise CheckError( + "no live required context transitively depends on security-gate; " + f"required={sorted(required)}" + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", type=Path, required=True) + parser.add_argument("--workflow", type=Path, default=Path(".github/workflows/ci.yml")) + parser.add_argument("--required-contexts", type=Path) + parser.add_argument("--results-json") + parser.add_argument("--event", default="pull_request") + parser.add_argument("--self-test-failure", action="store_true") + parser.add_argument("--self-test-unrequired", action="store_true") + args = parser.parse_args() + try: + matrix = load_json(args.matrix) + validate_matrix(matrix) + jobs = parse_jobs(args.workflow) + workflow_text = args.workflow.read_text() + validate_workflow(matrix, jobs, workflow_text) + if args.results_json is not None: + payload = json.loads(args.results_json) + if not isinstance(payload, dict): + raise CheckError("--results-json must be an object") + results = { + job: details.get("result", "missing") + if isinstance(details, dict) + else "missing" + for job, details in payload.items() + } + allowed_by_event = matrix.get("allowed_skips_by_event", {}) + allowed = set(allowed_by_event.get(args.event, [])) + if not evaluate(results, matrix["constituents"], allowed): + raise CheckError(f"security constituents failed for event {args.event}") + print("security-gate runtime results: PASS") + return 0 + if args.self_test_failure: + for state in ("failure", "cancelled", "timed_out"): + baseline = {job: "success" for job in matrix["constituents"]} + baseline[matrix["constituents"][0]] = state + if evaluate(baseline, matrix["constituents"], set()): + raise CheckError(f"{state} self-test unexpectedly passed") + missing = {job: "success" for job in matrix["constituents"][1:]} + if evaluate(missing, matrix["constituents"], set()): + raise CheckError("missing-result self-test unexpectedly passed") + print("security-gate failure self-test: PASS") + return 0 + if args.self_test_unrequired: + try: + validate_required_chain(matrix, jobs, {"contexts": ["Format"]}) + except CheckError: + print("security-gate unrequired-context self-test: PASS") + return 0 + raise CheckError("unrequired-context self-test unexpectedly passed") + if args.required_contexts is None: + raise CheckError("--required-contexts is required outside self-test modes") + validate_required_chain(matrix, jobs, load_json(args.required_contexts)) + except CheckError as exc: + print(f"security-gate check failed: {exc}", file=sys.stderr) + return 1 + print("security-gate contract: PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/qa-transports.sh b/scripts/qa-transports.sh new file mode 100755 index 00000000..8ef76430 --- /dev/null +++ b/scripts/qa-transports.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +contract="$repo_root/tests/fixtures/canonical_journey/contract.json" + +case "${1:-}" in + --self-test-harness) + rtk python3 - "$contract" <<'PY' +import json +import sys + +contract = json.load(open(sys.argv[1], encoding="utf-8")) +required = { + "initialize-discover", + "create-get-list-search-update", + "export-read-back", + "workspace-isolation", + "wrong-bearer", + "clean-shutdown", +} +assert contract["version"] == 1 +assert set(contract["transports"]) == {"stdio", "authenticated-http"} +assert required == set(contract["scenarios"]) +assert len(contract["required_tools"]) == len(set(contract["required_tools"])) +print("qa-transports harness: PASS") +PY + ;; + *) + echo "usage: $0 --self-test-harness" >&2 + exit 2 + ;; +esac diff --git a/scripts/smoke-example-module.py b/scripts/smoke-example-module.py new file mode 100755 index 00000000..b5dcb0d9 --- /dev/null +++ b/scripts/smoke-example-module.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Exercise an example module's dependency-free Engram adapter against a real server.""" + +from __future__ import annotations + +import importlib.util +import json +import pathlib +import sys + + +path = pathlib.Path(sys.argv[1]) +spec = importlib.util.spec_from_file_location("engram_example", path) +assert spec and spec.loader +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + +if hasattr(module, "remember_decision"): + created_raw = module.remember_decision(f"smoke decision from {path.parent.name}") + found_raw = module.search_memory("smoke decision") +else: + state = { + "query": "smoke decision", + "decision": f"smoke decision from {path.parent.name}", + "workspace": module.DEFAULT_WORKSPACE, + } + created_raw = module.remember_decision_node(state)["create_result"] + found_raw = module.search_memory_node(state)["search_result"] + +created = json.loads(created_raw) +found = json.loads(found_raw) +assert "error" not in created, created +assert "error" not in found, found +assert "result" in created and "result" in found diff --git a/scripts/test-canonical-journey.sh b/scripts/test-canonical-journey.sh new file mode 100755 index 00000000..fb21c3cc --- /dev/null +++ b/scripts/test-canonical-journey.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +exec rtk cargo test --test canonical_journey -- --nocapture diff --git a/scripts/test-examples.sh b/scripts/test-examples.sh new file mode 100755 index 00000000..20f365a0 --- /dev/null +++ b/scripts/test-examples.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" +export PYTHONDONTWRITEBYTECODE=1 + +if [[ "${1:-}" == "--self-test-broken-example" ]]; then + output="$(EXAMPLES_SELF_TEST_BROKEN=langgraph-tool bash "$0" 2>&1)" + status=$? + if [[ $status -eq 0 || "$output" != *"langgraph-tool"* ]]; then + printf 'self-test failed: aggregate did not report the broken langgraph-tool family\n' >&2 + exit 1 + fi + printf 'self-test passed: broken family was named and rejected\n' + exit 0 +fi + +tmp="$(mktemp -d "${TMPDIR:-/tmp}/engram-examples.XXXXXX")" +server_pid="" +cleanup() { + [[ -z "$server_pid" ]] || kill "$server_pid" 2>/dev/null || true + [[ -z "$server_pid" ]] || wait "$server_pid" 2>/dev/null || true + rm -rf "$tmp" +} +trap cleanup EXIT INT TERM + +failures=() +run_family() { + local family="$1" + shift + printf '[examples] %-22s ' "$family" + if [[ "${EXAMPLES_SELF_TEST_BROKEN:-}" == "$family" ]]; then + false + elif "$@" >"$tmp/$family.log" 2>&1; then + printf 'PASS\n' + return + fi + printf 'FAIL\n' >&2 + sed 's/^/ /' "$tmp/$family.log" >&2 2>/dev/null || true + failures+=("$family") +} + +python_payload_examples() { + python3 - <<'PY' +from pathlib import Path +for name in ( + "examples/openai-agents-sdk/agent_memory_tool.py", + "examples/fastmcp-server/server.py", + "examples/langgraph-tool/memory_graph.py", +): + compile(Path(name).read_text(), name, "exec") +PY + python3 examples/openai-agents-sdk/agent_memory_tool.py >/dev/null + python3 examples/fastmcp-server/server.py >/dev/null + python3 examples/langgraph-tool/memory_graph.py >/dev/null +} + +cursor_config() { + python3 - <<'PY' +import json, pathlib, re +text = pathlib.Path("examples/cursor-mcp/README.md").read_text() +blocks = re.findall(r"```json\n(.*?)\n```", text, re.S) +assert blocks, "missing Cursor JSON configuration" +config = json.loads(blocks[0]) +assert config["mcpServers"]["engram"]["args"] == ["--transport", "stdio"] +PY +} + +crewai_adapter() { + python3 - <<'PY' +from pathlib import Path +name = "sdks/python/engram_client/integrations/crewai.py" +compile(Path(name).read_text(), name, "exec") +PY + grep -q 'class EngramShortTermMemory' sdks/python/engram_client/integrations/crewai.py + grep -q 'class EngramLongTermMemory' sdks/python/engram_client/integrations/crewai.py + grep -q 'class EngramEntityMemory' sdks/python/engram_client/integrations/crewai.py +} + +run_family rust-library-demos cargo check --quiet --examples +run_family cursor-mcp cursor_config +run_family crewai-memory crewai_adapter +run_family python-patterns python_payload_examples + +# The network examples share one real, loopback-only server and disposable DB. +if cargo build --quiet --bin engram-server; then + port="$(python3 - <<'PY' +import socket +s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close() +PY +)" + token="examples-smoke-token" + ENGRAM_DB_PATH="$tmp/examples.db" ENGRAM_EMBEDDING_MODEL=tfidf \ + ENGRAM_HTTP_API_KEY="$token" target/debug/engram-server \ + --transport http --http-port "$port" >"$tmp/server.log" 2>&1 & + server_pid=$! + ready=0 + for _ in $(seq 1 100); do + if curl -fsS "http://127.0.0.1:$port/health" >/dev/null 2>&1; then ready=1; break; fi + kill -0 "$server_pid" 2>/dev/null || break + sleep 0.1 + done + if [[ $ready -eq 1 ]]; then + export ENGRAM_URL="http://127.0.0.1:$port/mcp" ENGRAM_HTTP_API_KEY="$token" + run_family claude-mcp bash examples/claude-mcp/seed-and-search.sh + for family in openai-agents-sdk fastmcp-server langgraph-tool; do + file="examples/$family/$(case "$family" in openai-agents-sdk) echo agent_memory_tool.py;; fastmcp-server) echo server.py;; *) echo memory_graph.py;; esac)" + run_family "$family" python3 scripts/smoke-example-module.py "$file" + done + else + failures+=("real-http-server") + printf '[examples] real-http-server FAIL\n' >&2 + sed 's/^/ /' "$tmp/server.log" >&2 + fi +else + failures+=("real-http-server") +fi + +if ((${#failures[@]})); then + printf 'example smoke failures: %s\n' "${failures[*]}" >&2 + exit 1 +fi +printf 'All example families passed without external credentials or services.\n' diff --git a/src/mcp/http_transport/mcp_handler.rs b/src/mcp/http_transport/mcp_handler.rs index f322583b..e5109fe4 100644 --- a/src/mcp/http_transport/mcp_handler.rs +++ b/src/mcp/http_transport/mcp_handler.rs @@ -1,7 +1,7 @@ use std::time::Instant; use axum::{ - extract::State, + extract::{ConnectInfo, State}, http::{HeaderMap, StatusCode, Uri}, response::{IntoResponse, Response}, Json, @@ -17,6 +17,7 @@ use super::{authenticate_transport_principal, AppState, MICROSECONDS_PER_MILLISE pub(super) async fn handle_mcp( State(state): State, + connect_info: Option>, headers: HeaderMap, uri: Uri, Json(request): Json, @@ -89,7 +90,9 @@ pub(super) async fn handle_mcp( }) }, ) - } else if !is_rate_limit_allowed(&state, &headers).await { + } else if !is_rate_limit_allowed(&state, &headers, connect_info.map(|info| info.0)) + .await + { is_rate_limited = true; decision = "rate_limited"; include_retry_after = true; diff --git a/src/mcp/http_transport/mod.rs b/src/mcp/http_transport/mod.rs index 4932c9bf..97369f2c 100644 --- a/src/mcp/http_transport/mod.rs +++ b/src/mcp/http_transport/mod.rs @@ -21,6 +21,7 @@ mod events; mod mcp_handler; mod rate_limit; mod router; +mod security_config; pub use router::serve_http; @@ -173,6 +174,7 @@ struct AppState { realtime: Option, rate_limiter: Option>>, metrics: Arc, + security: security_config::HttpSecurityConfig, } // --------------------------------------------------------------------------- diff --git a/src/mcp/http_transport/rate_limit.rs b/src/mcp/http_transport/rate_limit.rs index 81dfdee2..a2d1f477 100644 --- a/src/mcp/http_transport/rate_limit.rs +++ b/src/mcp/http_transport/rate_limit.rs @@ -60,7 +60,11 @@ pub(super) fn rate_limited_response( ) } -pub(super) async fn is_rate_limit_allowed(state: &AppState, headers: &HeaderMap) -> bool { +pub(super) async fn is_rate_limit_allowed( + state: &AppState, + headers: &HeaderMap, + peer: Option, +) -> bool { let rate_limiter = match &state.rate_limiter { Some(rate_limiter) => rate_limiter, None => return true, @@ -69,7 +73,7 @@ pub(super) async fn is_rate_limit_allowed(state: &AppState, headers: &HeaderMap) let now = Instant::now(); let mut limiter = rate_limiter.lock().await; let config = limiter.config.clone(); - let bucket_key = rate_limit_key(&config, headers); + let bucket_key = rate_limit_key(&config, headers, state.security.client_ip(peer, headers)); let decision = apply_rate_limit(&mut limiter, bucket_key, now); state @@ -147,7 +151,11 @@ pub(super) fn apply_rate_limit( } } -fn rate_limit_key(config: &RateLimiterConfig, headers: &HeaderMap) -> String { +fn rate_limit_key( + config: &RateLimiterConfig, + headers: &HeaderMap, + verified_ip: Option, +) -> String { if let Some(header_name) = config.key_header.as_deref() { if let Some(raw) = headers .get(header_name) @@ -165,22 +173,7 @@ fn rate_limit_key(config: &RateLimiterConfig, headers: &HeaderMap) -> String { } } - if let Some(xff) = headers - .get("x-forwarded-for") - .and_then(|header| header.to_str().ok()) - .and_then(|value| value.split(',').next()) - .map(|value| value.trim()) - .filter(|value| !value.is_empty()) - { - return format!("ip:{xff}"); - } - - if let Some(ip) = headers - .get("x-real-ip") - .and_then(|header| header.to_str().ok()) - .map(str::trim) - .filter(|value| !value.is_empty()) - { + if let Some(ip) = verified_ip { return format!("ip:{ip}"); } diff --git a/src/mcp/http_transport/router.rs b/src/mcp/http_transport/router.rs index db987faf..7f62d220 100644 --- a/src/mcp/http_transport/router.rs +++ b/src/mcp/http_transport/router.rs @@ -20,6 +20,7 @@ use super::mcp_handler::handle_mcp; use super::rate_limit::{ RateLimiterConfig, RateLimiterState, RATE_LIMIT_MAX_BUCKETS, RATE_LIMIT_STALE_AFTER_SECS, }; +use super::security_config::HttpSecurityConfig; use super::{normalize_api_key, AppState, HttpTransportMetrics}; use crate::realtime::RealtimeManager; @@ -150,6 +151,7 @@ pub(super) fn build_router( realtime, rate_limiter, metrics: Arc::new(HttpTransportMetrics::default()), + security: HttpSecurityConfig::from_env(), }; let cors = build_cors_layer(); @@ -202,7 +204,11 @@ pub async fn serve_http( let listener = tokio::net::TcpListener::bind(addr).await?; tracing::info!("HTTP transport listening on {}", addr); - axum::serve(listener, app).await?; + axum::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .await?; Ok(()) } diff --git a/src/mcp/http_transport/security_config.rs b/src/mcp/http_transport/security_config.rs new file mode 100644 index 00000000..f84f1a2a --- /dev/null +++ b/src/mcp/http_transport/security_config.rs @@ -0,0 +1,120 @@ +use std::net::{IpAddr, SocketAddr}; + +use axum::http::HeaderMap; + +const TRUSTED_PROXIES_ENV: &str = "ENGRAM_HTTP_TRUSTED_PROXIES"; +const MAX_FORWARDED_CHAIN: usize = 32; + +#[derive(Clone, Debug, Default)] +pub(super) struct HttpSecurityConfig { + trusted_proxies: Vec, +} + +impl HttpSecurityConfig { + pub(super) fn from_env() -> Self { + let trusted_proxies = std::env::var(TRUSTED_PROXIES_ENV) + .ok() + .map(|value| Self::parse_trusted_proxies(&value)) + .unwrap_or_default(); + Self { trusted_proxies } + } + + fn parse_trusted_proxies(value: &str) -> Vec { + value + .split(',') + .filter_map(|entry| match IpCidr::parse(entry.trim()) { + Some(cidr) => Some(cidr), + None => { + if !entry.trim().is_empty() { + tracing::warn!(value = %entry.trim(), "ignoring invalid trusted proxy CIDR"); + } + None + } + }) + .collect() + } + + pub(super) fn client_ip( + &self, + peer: Option, + headers: &HeaderMap, + ) -> Option { + let peer_ip = peer.map(|address| address.ip())?; + if !self + .trusted_proxies + .iter() + .any(|cidr| cidr.contains(peer_ip)) + { + return Some(peer_ip); + } + + let forwarded = match headers.get("x-forwarded-for").and_then(|v| v.to_str().ok()) { + Some(value) => value, + None => return Some(peer_ip), + }; + let chain: Vec = forwarded + .split(',') + .map(str::trim) + .map(str::parse) + .collect::>() + .ok() + .filter(|chain: &Vec| !chain.is_empty() && chain.len() <= MAX_FORWARDED_CHAIN) + .unwrap_or_default(); + if chain.is_empty() { + return Some(peer_ip); + } + + // Walk right-to-left across proxy hops. The first address outside the + // allowlist is the client. All-trusted chains normalize to the leftmost hop. + chain + .iter() + .rev() + .copied() + .find(|ip| !self.trusted_proxies.iter().any(|cidr| cidr.contains(*ip))) + .or_else(|| chain.first().copied()) + .or(Some(peer_ip)) + } +} + +#[derive(Clone, Copy, Debug)] +struct IpCidr { + network: IpAddr, + prefix: u8, +} + +impl IpCidr { + fn parse(value: &str) -> Option { + let (ip, prefix): (IpAddr, u8) = match value.split_once('/') { + Some((ip, prefix)) => (ip.parse().ok()?, prefix.parse().ok()?), + None => { + let ip: IpAddr = value.parse().ok()?; + let prefix = if ip.is_ipv4() { 32 } else { 128 }; + return Some(Self { + network: ip, + prefix, + }); + } + }; + let max = if ip.is_ipv4() { 32 } else { 128 }; + (prefix <= max).then_some(Self { + network: ip, + prefix, + }) + } + + fn contains(self, ip: IpAddr) -> bool { + match (self.network, ip) { + (IpAddr::V4(network), IpAddr::V4(ip)) => { + let mask = u32::MAX.checked_shl((32 - self.prefix).into()).unwrap_or(0); + u32::from(network) & mask == u32::from(ip) & mask + } + (IpAddr::V6(network), IpAddr::V6(ip)) => { + let mask = u128::MAX + .checked_shl((128 - self.prefix).into()) + .unwrap_or(0); + u128::from(network) & mask == u128::from(ip) & mask + } + _ => false, + } + } +} diff --git a/src/mcp/http_transport/tests/rate_limit_keying.rs b/src/mcp/http_transport/tests/rate_limit_keying.rs index 9c39eb86..2b93d454 100644 --- a/src/mcp/http_transport/tests/rate_limit_keying.rs +++ b/src/mcp/http_transport/tests/rate_limit_keying.rs @@ -40,7 +40,7 @@ async fn test_post_mcp_rate_limit_uses_custom_key_header() { } #[tokio::test] -async fn test_post_mcp_rate_limit_uses_x_real_ip_fallback() { +async fn test_post_mcp_rate_limit_ignores_unverified_x_real_ip() { let app = test_app_with_rate_limits(Some("secret-key"), 100, 1, None); let first_ip = app @@ -73,11 +73,11 @@ async fn test_post_mcp_rate_limit_uses_x_real_ip_fallback() { )) .await .expect("request should be handled"); - assert_eq!(second_ip.status(), StatusCode::OK); + assert_eq!(second_ip.status(), StatusCode::TOO_MANY_REQUESTS); } #[tokio::test] -async fn test_post_mcp_rate_limit_prefers_x_forwarded_for_over_x_real_ip() { +async fn test_post_mcp_rate_limit_ignores_unverified_forwarding_headers() { let app = test_app_with_rate_limits(Some("secret-key"), 100, 1, None); let xff_first = app @@ -122,7 +122,10 @@ async fn test_post_mcp_rate_limit_prefers_x_forwarded_for_over_x_real_ip() { )) .await .expect("request should be handled"); - assert_eq!(different_xff_same_real_ip.status(), StatusCode::OK); + assert_eq!( + different_xff_same_real_ip.status(), + StatusCode::TOO_MANY_REQUESTS + ); } #[tokio::test] @@ -168,5 +171,5 @@ async fn test_post_mcp_rate_limit_empty_key_disables_header_keying() { )) .await .expect("request should be handled"); - assert_eq!(tenant_b.status(), StatusCode::OK); + assert_eq!(tenant_b.status(), StatusCode::TOO_MANY_REQUESTS); } diff --git a/tests/canonical_journey.rs b/tests/canonical_journey.rs new file mode 100644 index 00000000..a768601e --- /dev/null +++ b/tests/canonical_journey.rs @@ -0,0 +1,286 @@ +mod support; + +use std::fs; +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::path::Path; +use std::sync::Mutex; +use std::time::Duration; + +use serde_json::{json, Value}; +use support::real_server::{ + bad_executable_path, initialize_request, tool_call_request, tool_result_json, + tools_list_request, RealServer, RealServerConfig, +}; + +const WORKSPACE: &str = "canonical-journey"; +const OTHER_WORKSPACE: &str = "canonical-private"; +const ORIGINAL: &str = "Canonical journey remembers the cobalt launch checklist"; +const UPDATED: &str = "Canonical journey remembers the updated cobalt launch checklist"; +const PRIVATE: &str = "private workspace sentinel must never cross the boundary"; +static JOURNEY_LOCK: Mutex<()> = Mutex::new(()); + +enum Transport { + Stdio, + Http, +} + +#[test] +fn canonical_real_binary_journey_over_stdio_and_authenticated_http() { + let _guard = JOURNEY_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + + let state = tempfile::tempdir().expect("create shared canonical journey state"); + let db_path = state.path().join("canonical-journey.db"); + + let stdio_id = run_journey(Transport::Stdio, &db_path, None); + run_journey(Transport::Http, &db_path, Some(stdio_id)); + + let state_path = state.path().to_path_buf(); + drop(state); + assert!(!state_path.exists(), "shared journey state was not removed"); +} + +fn run_journey(transport: Transport, db_path: &Path, existing_id: Option) -> i64 { + let config = RealServerConfig::from_cargo_bin(); + let api_key = config.api_key().to_string(); + // Keep every public Todo 10 harness contract exercised by this integration target so + // target-scoped Clippy sees the shared support module as fully used. + let _bad_config_contract = + RealServerConfig::from_cargo_bin().with_executable(bad_executable_path()); + let _managed_stdio_contract = RealServer::start_stdio; + let _managed_stdio_in_contract = RealServer::start_stdio_in; + let _managed_http_contract = RealServer::start_http; + let mut server = match transport { + Transport::Stdio => RealServer::start_stdio_at(config, db_path.to_path_buf()) + .expect("start canonical stdio server"), + Transport::Http => RealServer::start_http_at(config, db_path.to_path_buf()) + .expect("start canonical HTTP server"), + }; + let temp_path = server.temp_path().to_path_buf(); + assert_eq!(server.db_path(), db_path); + + let initialized = request(&mut server, &transport, initialize_request(1)); + assert_eq!(initialized["result"]["protocolVersion"], "2025-11-25"); + + let tools = request(&mut server, &transport, tools_list_request(2)); + for name in fixture_tools() { + assert_has_tool(&tools, &name); + } + + if let Some(id) = existing_id { + let persisted = call(&mut server, &transport, 12, "memory_get", json!({"id": id})); + assert_eq!(persisted["content"], UPDATED); + } + + let created = call( + &mut server, + &transport, + 3, + "memory_create", + json!({"content": ORIGINAL, "workspace": WORKSPACE, "tags": ["canonical"]}), + ); + let id = created["id"] + .as_i64() + .expect("memory_create returns numeric id"); + assert_eq!(created["content"], ORIGINAL); + + let private = call( + &mut server, + &transport, + 4, + "memory_create", + json!({"content": PRIVATE, "workspace": OTHER_WORKSPACE}), + ); + assert_ne!(private["id"], id); + + let fetched = call(&mut server, &transport, 5, "memory_get", json!({"id": id})); + assert_eq!(fetched["content"], ORIGINAL); + + let listed = call( + &mut server, + &transport, + 6, + "memory_list", + json!({"workspace": WORKSPACE}), + ); + assert_contains_id(&listed, id); + assert_not_contains(&listed, PRIVATE); + + let searched = call( + &mut server, + &transport, + 7, + "memory_search", + json!({"query": "cobalt launch checklist", "workspace": WORKSPACE, "rerank": false}), + ); + assert_contains_id(&searched, id); + assert_not_contains(&searched, PRIVATE); + + let updated = call( + &mut server, + &transport, + 8, + "memory_update", + json!({"id": id, "content": UPDATED}), + ); + assert_eq!(updated["content"], UPDATED); + + let exported = call( + &mut server, + &transport, + 9, + "memory_export", + json!({"workspace": WORKSPACE}), + ); + assert_contains_id(&exported, id); + assert_not_contains(&exported, PRIVATE); + + let export_dir = temp_path.join("markdown-readback"); + let markdown = call( + &mut server, + &transport, + 10, + "memory_export_markdown", + json!({"workspace": WORKSPACE, "output_dir": export_dir, "include_links": false}), + ); + assert!(markdown["files_written"].as_u64().unwrap_or_default() >= 2); + assert_markdown_readback(&export_dir, UPDATED); + + let isolated = call( + &mut server, + &transport, + 11, + "memory_list", + json!({"workspace": "canonical-empty"}), + ); + assert_not_contains(&isolated, ORIGINAL); + assert_not_contains(&isolated, UPDATED); + assert_not_contains(&isolated, PRIVATE); + + if let Some(port) = server.port() { + assert_wrong_bearer_rejected(port, &api_key); + } + + let cleanup = server.shutdown_and_verify(); + assert!(cleanup.child_id > 0); + assert_eq!(cleanup.temp_path, temp_path); + assert_eq!(cleanup.port.is_some(), matches!(transport, Transport::Http)); + assert!( + !cleanup.temp_removed, + "server must not remove caller-owned shared state" + ); + assert!( + cleanup.port_released.unwrap_or(true), + "HTTP port was not released" + ); + assert!(!cleanup.redacted_stderr.contains(&api_key)); + assert!( + temp_path.exists(), + "shared state disappeared before both transports completed" + ); + id +} + +fn request(server: &mut RealServer, transport: &Transport, value: Value) -> Value { + match transport { + Transport::Stdio => server + .stdio_request(value) + .expect("stdio JSON-RPC response"), + Transport::Http => server.http_json_rpc(value).expect("HTTP JSON-RPC response"), + } +} + +fn call( + server: &mut RealServer, + transport: &Transport, + id: i64, + name: &str, + arguments: Value, +) -> Value { + let response = request(server, transport, tool_call_request(id, name, arguments)); + tool_result_json(&response) +} + +fn fixture_tools() -> Vec { + let fixture: Value = + serde_json::from_str(include_str!("fixtures/canonical_journey/contract.json")) + .expect("canonical journey contract fixture is valid JSON"); + fixture["required_tools"] + .as_array() + .expect("required_tools array") + .iter() + .map(|value| value.as_str().expect("tool name string").to_string()) + .collect() +} + +fn assert_has_tool(response: &Value, expected: &str) { + let tools = response["result"]["tools"].as_array().expect("tools array"); + assert!(tools.iter().any(|tool| tool["name"] == expected)); +} + +fn assert_contains_id(value: &Value, id: i64) { + assert!( + contains_id(value, id), + "response does not contain id {id}: {value}" + ); +} + +fn contains_id(value: &Value, id: i64) -> bool { + match value { + Value::Object(map) => { + map.get("id").and_then(Value::as_i64) == Some(id) + || map.values().any(|child| contains_id(child, id)) + } + Value::Array(values) => values.iter().any(|child| contains_id(child, id)), + _ => false, + } +} + +fn assert_not_contains(value: &Value, forbidden: &str) { + assert!( + !value.to_string().contains(forbidden), + "workspace data leaked: {value}" + ); +} + +fn assert_markdown_readback(root: &Path, expected: &str) { + let mut found = false; + for entry in fs::read_dir(root).expect("read markdown export directory") { + let path = entry.expect("read markdown entry").path(); + if path.extension().and_then(|ext| ext.to_str()) == Some("md") { + let text = fs::read_to_string(path).expect("read exported markdown"); + found |= text.contains(expected); + assert!(!text.contains(PRIVATE)); + } + } + assert!(found, "updated content missing from markdown export"); +} + +fn assert_wrong_bearer_rejected(port: u16, valid_key: &str) { + let body = initialize_request(99).to_string(); + let wrong = format!("{valid_key}-wrong"); + let mut stream = TcpStream::connect(("127.0.0.1", port)).expect("connect with wrong bearer"); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .expect("set timeout"); + write!( + stream, + "POST /mcp HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nAuthorization: Bearer {wrong}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .expect("write wrong bearer request"); + stream.flush().expect("flush wrong bearer request"); + let mut response = String::new(); + stream + .read_to_string(&mut response) + .expect("read rejection"); + assert!( + response.starts_with("HTTP/1.1 401"), + "wrong bearer was not rejected: {response}" + ); + assert!(!response.contains(ORIGINAL)); + assert!(!response.contains(UPDATED)); + assert!(!response.contains(PRIVATE)); +} diff --git a/tests/fixtures/canonical_journey/contract.json b/tests/fixtures/canonical_journey/contract.json new file mode 100644 index 00000000..5ac77961 --- /dev/null +++ b/tests/fixtures/canonical_journey/contract.json @@ -0,0 +1,21 @@ +{ + "version": 1, + "required_tools": [ + "memory_create", + "memory_get", + "memory_list", + "memory_search", + "memory_update", + "memory_export", + "memory_export_markdown" + ], + "scenarios": [ + "initialize-discover", + "create-get-list-search-update", + "export-read-back", + "workspace-isolation", + "wrong-bearer", + "clean-shutdown" + ], + "transports": ["stdio", "authenticated-http"] +} diff --git a/tests/fixtures/retrieval_quality/baseline.json b/tests/fixtures/retrieval_quality/baseline.json new file mode 100644 index 00000000..53907dd8 --- /dev/null +++ b/tests/fixtures/retrieval_quality/baseline.json @@ -0,0 +1,40 @@ +{ + "schema_version": "engram.quality-baseline.v1", + "source_revision": "81be152c713230c082901899e6880579fcedabb3", + "generated_at": "2026-07-12T00:00:00Z", + "deterministic_seed": 20260712, + "corpus": { + "name": "engram-synthetic-representative", + "version": "1.0.0", + "fixture_path": "tests/fixtures/retrieval_quality/corpus.json", + "schema": { + "memory_fields": [ + "key", + "workspace", + "content" + ], + "query_fields": [ + "key", + "query", + "workspace", + "fuzzy", + "forbidden_workspaces" + ], + "relevance_fields": [ + "memory_key", + "grade" + ] + }, + "memory_count": 9, + "query_count": 6 + }, + "metrics": { + "recall@10": 1.0, + "mrr": 1.0, + "ndcg@10": 1.0 + }, + "benchmark_evidence": { + "criterion_baseline": "benches/results/benchmark_baseline.txt", + "dream_eval_runbook": "docs/DREAM_SNAPSHOT_EVALS.md" + } +} diff --git a/tests/fixtures/retrieval_quality/corpus.json b/tests/fixtures/retrieval_quality/corpus.json new file mode 100644 index 00000000..fd5bde8f --- /dev/null +++ b/tests/fixtures/retrieval_quality/corpus.json @@ -0,0 +1,27 @@ +{ + "schema_version": "engram.retrieval-corpus.v1", + "name": "engram-synthetic-representative", + "version": "1.0.0", + "license": "CC0-1.0", + "source": "Synthetic corpus authored for Engram; no private data or PII.", + "deterministic_seed": 20260712, + "memories": [ + {"key":"exact-rust","workspace":"alpha","content":"The Rust release checklist requires cargo clippy before submission."}, + {"key":"synonym-auth","workspace":"alpha","content":"Authentication login uses bearer credentials; the access token is rotated monthly."}, + {"key":"fuzzy-backup","workspace":"alpha","content":"The backup procedure stores encrypted snapshots in cold storage."}, + {"key":"temporal-july","workspace":"alpha","content":"On 2026-07-04 the Atlas migration decision moved indexing to Friday."}, + {"key":"identity-alice","workspace":"alpha","content":"Alice Nguyen owns the incident response runbook and escalation policy."}, + {"key":"distractor-coffee","workspace":"alpha","content":"The office coffee order includes decaf beans and paper filters."}, + {"key":"distractor-ui","workspace":"alpha","content":"The dashboard color palette uses cobalt and neutral gray."}, + {"key":"workspace-secret","workspace":"beta","content":"Orchid launch cipher is restricted to the beta workspace."}, + {"key":"workspace-alpha-distractor","workspace":"alpha","content":"Orchid flowers appear in the lobby artwork, without launch information."} + ], + "queries": [ + {"key":"exact","query":"Rust release checklist","workspace":"alpha","relevance":{"exact-rust":3}}, + {"key":"synonym","query":"login credential","workspace":"alpha","relevance":{"synonym-auth":3}}, + {"key":"typo-fuzzy","query":"bakcup procedure","workspace":"alpha","fuzzy":true,"relevance":{"fuzzy-backup":3}}, + {"key":"temporal","query":"2026-07-04 Atlas migration","workspace":"alpha","relevance":{"temporal-july":3}}, + {"key":"identity","query":"Alice Nguyen incident response","workspace":"alpha","relevance":{"identity-alice":3}}, + {"key":"workspace-isolation","query":"Orchid launch cipher","workspace":"beta","relevance":{"workspace-secret":3},"forbidden_workspaces":["alpha"]} + ] +} diff --git a/tests/fixtures/security_gate_matrix.json b/tests/fixtures/security_gate_matrix.json new file mode 100644 index 00000000..526e4596 --- /dev/null +++ b/tests/fixtures/security_gate_matrix.json @@ -0,0 +1,52 @@ +{ + "schema_version": 1, + "aggregate_job": "security-gate", + "required_dependency_job": "test", + "constituents": [ + "audit", + "deny", + "security-exceptions", + "codeql-security", + "semgrep-security", + "gitleaks-security", + "agentshield-security" + ], + "allowed_skips_by_event": { + "pull_request": [], + "push": [], + "schedule": [], + "workflow_dispatch": [], + "not_applicable_fixture": ["semgrep-security"] + }, + "scenarios": [ + { + "name": "all-security-checks-pass", + "results": { + "audit": "success", + "deny": "success", + "security-exceptions": "success", + "codeql-security": "success", + "semgrep-security": "success", + "gitleaks-security": "success", + "agentshield-security": "success" + }, + "allowed_skips": [], + "expected": "success" + }, + { + "name": "legitimate-event-skip-is-neutral", + "event": "not_applicable_fixture", + "results": { + "audit": "success", + "deny": "success", + "security-exceptions": "success", + "codeql-security": "success", + "semgrep-security": "skipped", + "gitleaks-security": "success", + "agentshield-security": "success" + }, + "allowed_skips": ["semgrep-security"], + "expected": "success" + } + ] +} diff --git a/tests/http_transport_security.rs b/tests/http_transport_security.rs index 2c3a2b4c..5b8eb06f 100644 --- a/tests/http_transport_security.rs +++ b/tests/http_transport_security.rs @@ -191,6 +191,168 @@ fn public_http_with_key_authenticates_mcp_and_sse() { ); } +#[test] +fn trusted_proxy_spoofed_forwarding_is_ignored_by_rate_key() { + let _guard = HTTP_SECURITY_TEST_LOCK.lock().expect("test lock"); + let port = pick_loopback_port(); + let mut process = ServerProcess::spawn(&[ + "--transport", + "http", + "--http-port", + &port.to_string(), + "--http-api-key", + "secret-key", + "--http-rate-limit-rps", + "1", + "--http-rate-limit-burst", + "1", + ]) + .expect("spawn server"); + process + .wait_for_log(&format!("HTTP transport listening on 127.0.0.1:{port}")) + .expect("readiness"); + + assert_eq!( + http_post_json_with_headers( + port, + "/mcp", + initialize_request(), + Some("secret-key"), + &[("X-Forwarded-For", "198.51.100.1")] + ) + .unwrap() + .status, + 200 + ); + assert_eq!( + http_post_json_with_headers( + port, + "/mcp", + initialize_request(), + Some("secret-key"), + &[("X-Forwarded-For", "198.51.100.2")] + ) + .unwrap() + .status, + 429 + ); +} + +#[test] +fn trusted_proxy_cidr_honors_and_normalizes_forwarded_chain() { + let _guard = HTTP_SECURITY_TEST_LOCK.lock().expect("test lock"); + let port = pick_loopback_port(); + let mut process = ServerProcess::spawn_with_env( + &[ + "--transport", + "http", + "--http-port", + &port.to_string(), + "--http-api-key", + "secret-key", + "--http-rate-limit-rps", + "1", + "--http-rate-limit-burst", + "1", + ], + &[("ENGRAM_HTTP_TRUSTED_PROXIES", "127.0.0.0/8,10.0.0.0/8")], + ) + .expect("spawn server"); + process + .wait_for_log(&format!("HTTP transport listening on 127.0.0.1:{port}")) + .expect("readiness"); + + assert_eq!( + http_post_json_with_headers( + port, + "/mcp", + initialize_request(), + Some("secret-key"), + &[("X-Forwarded-For", "198.51.100.1, 10.1.2.3")] + ) + .unwrap() + .status, + 200 + ); + assert_eq!( + http_post_json_with_headers( + port, + "/mcp", + initialize_request(), + Some("secret-key"), + &[("X-Forwarded-For", "198.51.100.2")] + ) + .unwrap() + .status, + 200 + ); + assert_eq!( + http_post_json_with_headers( + port, + "/mcp", + initialize_request(), + Some("secret-key"), + &[("X-Forwarded-For", "198.51.100.1")] + ) + .unwrap() + .status, + 429 + ); +} + +#[test] +fn trusted_proxy_malformed_or_overlong_chain_falls_back_to_peer() { + let _guard = HTTP_SECURITY_TEST_LOCK.lock().expect("test lock"); + let port = pick_loopback_port(); + let mut process = ServerProcess::spawn_with_env( + &[ + "--transport", + "http", + "--http-port", + &port.to_string(), + "--http-api-key", + "secret-key", + "--http-rate-limit-rps", + "1", + "--http-rate-limit-burst", + "1", + ], + &[("ENGRAM_HTTP_TRUSTED_PROXIES", "127.0.0.0/8")], + ) + .expect("spawn server"); + process + .wait_for_log(&format!("HTTP transport listening on 127.0.0.1:{port}")) + .expect("readiness"); + + assert_eq!( + http_post_json_with_headers( + port, + "/mcp", + initialize_request(), + Some("secret-key"), + &[("X-Forwarded-For", "not-an-ip")] + ) + .unwrap() + .status, + 200 + ); + let overlong = std::iter::repeat_n("198.51.100.1", 33) + .collect::>() + .join(","); + assert_eq!( + http_post_json_with_headers( + port, + "/mcp", + initialize_request(), + Some("secret-key"), + &[("X-Forwarded-For", &overlong)] + ) + .unwrap() + .status, + 429 + ); +} + struct ServerProcess { child: Child, _temp_dir: TempDir, @@ -200,9 +362,14 @@ struct ServerProcess { impl ServerProcess { fn spawn(args: &[&str]) -> std::io::Result { + Self::spawn_with_env(args, &[]) + } + + fn spawn_with_env(args: &[&str], env: &[(&str, &str)]) -> std::io::Result { let temp_dir = tempfile::tempdir()?; let mut command = base_command_in_tempdir(&temp_dir); command.args(args); + command.envs(env.iter().copied()); command .stdin(Stdio::null()) .stdout(Stdio::null()) @@ -286,13 +453,27 @@ fn http_post_json( path: &str, body: serde_json::Value, bearer: Option<&str>, +) -> Result { + http_post_json_with_headers(port, path, body, bearer, &[]) +} + +fn http_post_json_with_headers( + port: u16, + path: &str, + body: serde_json::Value, + bearer: Option<&str>, + headers: &[(&str, &str)], ) -> Result { let body = body.to_string(); let auth = bearer .map(|token| format!("Authorization: Bearer {token}\r\n")) .unwrap_or_default(); + let extra_headers = headers + .iter() + .map(|(name, value)| format!("{name}: {value}\r\n")) + .collect::(); let request = format!( - "POST {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n{auth}Connection: close\r\n\r\n{body}", + "POST {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n{auth}{extra_headers}Connection: close\r\n\r\n{body}", body.len() ); http_request(port, &request) diff --git a/tests/retrieval_quality.rs b/tests/retrieval_quality.rs new file mode 100644 index 00000000..23e3e2c2 --- /dev/null +++ b/tests/retrieval_quality.rs @@ -0,0 +1,315 @@ +//! Deterministic retrieval-quality evaluation over a frozen synthetic corpus. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; + +use engram::embedding::{create_embedder, EmbeddingCache}; +use engram::mcp::handlers::{self, HandlerContext}; +use engram::search::{AdaptiveCacheConfig, FuzzyEngine, SearchConfig, SearchResultCache}; +use engram::storage::queries::create_memory; +use engram::storage::Storage; +use engram::types::{CreateMemoryInput, EmbeddingConfig}; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +const CORPUS_JSON: &str = include_str!("fixtures/retrieval_quality/corpus.json"); +const BASELINE_JSON: &str = include_str!("fixtures/retrieval_quality/baseline.json"); + +#[derive(Clone, Deserialize)] +struct Corpus { + schema_version: String, + name: String, + version: String, + license: String, + source: String, + deterministic_seed: u64, + memories: Vec, + queries: Vec, +} + +#[derive(Clone, Deserialize)] +struct FixtureMemory { + key: String, + workspace: String, + content: String, +} + +#[derive(Clone, Deserialize)] +struct FixtureQuery { + key: String, + query: String, + workspace: String, + #[serde(default)] + fuzzy: bool, + relevance: BTreeMap, + #[serde(default)] + forbidden_workspaces: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +struct Metrics { + #[serde(rename = "recall@10")] + recall_at_10: f64, + mrr: f64, + #[serde(rename = "ndcg@10")] + ndcg_at_10: f64, +} + +#[derive(Serialize)] +struct Baseline<'a> { + schema_version: &'a str, + source_revision: &'a str, + generated_at: &'a str, + deterministic_seed: u64, + corpus: BaselineCorpus<'a>, + metrics: Metrics, + benchmark_evidence: BenchmarkEvidence<'a>, +} + +#[derive(Serialize)] +struct BaselineCorpus<'a> { + name: &'a str, + version: &'a str, + fixture_path: &'a str, + schema: FixtureSchema, + memory_count: usize, + query_count: usize, +} + +#[derive(Serialize)] +struct FixtureSchema { + memory_fields: [&'static str; 3], + query_fields: [&'static str; 5], + relevance_fields: [&'static str; 2], +} + +#[derive(Serialize)] +struct BenchmarkEvidence<'a> { + criterion_baseline: &'a str, + dream_eval_runbook: &'a str, +} + +fn context() -> HandlerContext { + HandlerContext { + storage: Storage::open_in_memory().expect("in-memory storage"), + embedder: create_embedder(&EmbeddingConfig::default()).expect("deterministic embedder"), + fuzzy_engine: Arc::new(Mutex::new(FuzzyEngine::new())), + search_config: SearchConfig::default(), + realtime: None, + embedding_cache: Arc::new(EmbeddingCache::default()), + search_cache: Arc::new(SearchResultCache::new(AdaptiveCacheConfig::default())), + #[cfg(feature = "meilisearch")] + meili: None, + #[cfg(feature = "meilisearch")] + meili_indexer: None, + #[cfg(feature = "meilisearch")] + meili_sync_interval: 60, + #[cfg(feature = "langfuse")] + langfuse_runtime: Arc::new(tokio::runtime::Runtime::new().expect("langfuse runtime")), + } +} + +fn evaluate(corpus: &Corpus) -> Result> { + let ctx = context(); + let mut ids = HashMap::new(); + + for memory in &corpus.memories { + let created = ctx + .storage + .with_transaction(|conn| { + create_memory( + conn, + &CreateMemoryInput { + content: memory.content.clone(), + workspace: Some(memory.workspace.clone()), + ..Default::default() + }, + ) + }) + .expect("fixture memory must be valid"); + ids.insert(created.id, memory.key.clone()); + ctx.fuzzy_engine.lock().add_to_vocabulary(&memory.content); + } + + let mut errors = Vec::new(); + let mut recall_sum = 0.0; + let mut reciprocal_rank_sum = 0.0; + let mut ndcg_sum = 0.0; + + for query in &corpus.queries { + let effective_query = if query.fuzzy { + ctx.fuzzy_engine + .lock() + .correct_query(&query.query) + .corrected_query + .unwrap_or_else(|| query.query.clone()) + } else { + query.query.clone() + }; + let response = handlers::dispatch( + &ctx, + "memory_search", + json!({ + "query": effective_query, + "workspace": query.workspace, + "limit": 10, + "rerank": false, + "skip_cache": true + }), + ); + let results = result_array(&response).unwrap_or_else(|| { + errors.push(format!("{}: search returned {response}", query.key)); + &[] + }); + + let ranked: Vec<(String, String)> = results + .iter() + .filter_map(|result| { + let memory = result.get("memory")?; + let id = memory.get("id")?.as_i64()?; + let workspace = memory.get("workspace")?.as_str()?.to_owned(); + Some((ids.get(&id)?.clone(), workspace)) + }) + .collect(); + + for (key, workspace) in &ranked { + if query + .forbidden_workspaces + .iter() + .any(|item| item == workspace) + { + errors.push(format!( + "{}: forbidden workspace {workspace} leaked through result {key}", + query.key + )); + } + } + + let relevant: HashSet<&str> = query.relevance.keys().map(String::as_str).collect(); + let found = ranked + .iter() + .take(10) + .filter(|(key, _)| relevant.contains(key.as_str())) + .count(); + if found != relevant.len() { + errors.push(format!( + "{}: retrieved {found}/{} relevant memories; ranking={ranked:?}", + query.key, + relevant.len() + )); + } + recall_sum += found as f64 / relevant.len() as f64; + + if let Some(rank) = ranked + .iter() + .position(|(key, _)| relevant.contains(key.as_str())) + { + reciprocal_rank_sum += 1.0 / (rank + 1) as f64; + } + + let dcg: f64 = ranked + .iter() + .take(10) + .enumerate() + .map(|(rank, (key, _))| { + let grade = f64::from(*query.relevance.get(key).unwrap_or(&0)); + (2_f64.powf(grade) - 1.0) / ((rank + 2) as f64).log2() + }) + .sum(); + let mut grades: Vec = query.relevance.values().copied().collect(); + grades.sort_unstable_by(|a, b| b.cmp(a)); + let ideal: f64 = grades + .iter() + .take(10) + .enumerate() + .map(|(rank, grade)| (2_f64.powf(f64::from(*grade)) - 1.0) / ((rank + 2) as f64).log2()) + .sum(); + ndcg_sum += if ideal == 0.0 { 0.0 } else { dcg / ideal }; + } + + if !errors.is_empty() { + return Err(errors); + } + let count = corpus.queries.len() as f64; + Ok(Metrics { + recall_at_10: recall_sum / count, + mrr: reciprocal_rank_sum / count, + ndcg_at_10: ndcg_sum / count, + }) +} + +fn result_array(response: &Value) -> Option<&[Value]> { + response + .as_array() + .or_else(|| response.get("results")?.as_array()) + .map(Vec::as_slice) +} + +#[test] +fn frozen_corpus_matches_byte_identical_baseline() { + let corpus: Corpus = serde_json::from_str(CORPUS_JSON).expect("valid corpus fixture"); + assert_eq!(corpus.schema_version, "engram.retrieval-corpus.v1"); + assert_eq!(corpus.license, "CC0-1.0"); + assert!(corpus.source.contains("no private data or PII")); + + let metrics = evaluate(&corpus).unwrap_or_else(|errors| panic!("{}", errors.join("\n"))); + let baseline = Baseline { + schema_version: "engram.quality-baseline.v1", + source_revision: "81be152c713230c082901899e6880579fcedabb3", + generated_at: "2026-07-12T00:00:00Z", + deterministic_seed: corpus.deterministic_seed, + corpus: BaselineCorpus { + name: &corpus.name, + version: &corpus.version, + fixture_path: "tests/fixtures/retrieval_quality/corpus.json", + schema: FixtureSchema { + memory_fields: ["key", "workspace", "content"], + query_fields: ["key", "query", "workspace", "fuzzy", "forbidden_workspaces"], + relevance_fields: ["memory_key", "grade"], + }, + memory_count: corpus.memories.len(), + query_count: corpus.queries.len(), + }, + metrics, + benchmark_evidence: BenchmarkEvidence { + criterion_baseline: "benches/results/benchmark_baseline.txt", + dream_eval_runbook: "docs/DREAM_SNAPSHOT_EVALS.md", + }, + }; + let emitted = serde_json::to_string_pretty(&baseline).expect("serialize baseline") + "\n"; + assert_eq!( + emitted, BASELINE_JSON, + "regenerate and review baseline drift" + ); + println!("{emitted}"); +} + +#[test] +fn evaluator_reports_missing_relevant_memory() { + let mut corpus: Corpus = serde_json::from_str(CORPUS_JSON).expect("valid corpus fixture"); + corpus.memories.retain(|memory| memory.key != "exact-rust"); + let errors = evaluate(&corpus).expect_err("missing relevant memory must fail"); + assert!(errors + .iter() + .any(|error| error.contains("exact: retrieved 0/1"))); +} + +#[test] +fn evaluator_reports_cross_workspace_leak() { + let mut corpus: Corpus = serde_json::from_str(CORPUS_JSON).expect("valid corpus fixture"); + let isolation = corpus + .queries + .iter_mut() + .find(|query| query.key == "workspace-isolation") + .expect("workspace isolation query"); + isolation.workspace = "alpha".to_owned(); + isolation.query = "Orchid launch".to_owned(); + isolation + .relevance + .insert("workspace-alpha-distractor".to_owned(), 1); + let errors = evaluate(&corpus).expect_err("forbidden workspace must fail"); + assert!(errors + .iter() + .any(|error| error.contains("forbidden workspace alpha leaked"))); +} diff --git a/tests/support/real_server/process.rs b/tests/support/real_server/process.rs index b1087d67..98293c88 100644 --- a/tests/support/real_server/process.rs +++ b/tests/support/real_server/process.rs @@ -41,6 +41,19 @@ impl RealServer { pub fn start_stdio_in(config: RealServerConfig, temp_dir: TempDir) -> Result { let db_path = temp_dir.path().join("stdio-memories.db"); + Self::start_stdio_with_state(config, db_path, Some(temp_dir)) + } + + #[allow(dead_code)] // Shared support is compiled independently by tests that do not reuse state. + pub fn start_stdio_at(config: RealServerConfig, db_path: PathBuf) -> Result { + Self::start_stdio_with_state(config, db_path, None) + } + + fn start_stdio_with_state( + config: RealServerConfig, + db_path: PathBuf, + temp_dir: Option, + ) -> Result { let mut command = base_command(&config, &db_path); command.arg("--transport").arg("stdio"); command @@ -63,7 +76,7 @@ impl RealServer { Ok(Self { child, - temp_dir: Some(temp_dir), + temp_dir, db_path, api_key: config.api_key, port: None, @@ -77,6 +90,19 @@ impl RealServer { pub fn start_http(config: RealServerConfig) -> Result { let temp_dir = tempfile::tempdir().map_err(start_error)?; let db_path = temp_dir.path().join("http-memories.db"); + Self::start_http_with_state(config, db_path, Some(temp_dir)) + } + + #[allow(dead_code)] // Shared support is compiled independently by tests that do not reuse state. + pub fn start_http_at(config: RealServerConfig, db_path: PathBuf) -> Result { + Self::start_http_with_state(config, db_path, None) + } + + fn start_http_with_state( + config: RealServerConfig, + db_path: PathBuf, + temp_dir: Option, + ) -> Result { let port = pick_loopback_port().map_err(start_error)?; let mut command = base_command(&config, &db_path); command @@ -99,7 +125,7 @@ impl RealServer { let executable = config.executable().to_path_buf(); let mut server = Self { child, - temp_dir: Some(temp_dir), + temp_dir, db_path, api_key: config.api_key, port: Some(port), @@ -113,10 +139,10 @@ impl RealServer { } pub fn temp_path(&self) -> &Path { - self.temp_dir - .as_ref() - .expect("server temp dir should be present while running") - .path() + self.temp_dir.as_ref().map_or_else( + || self.db_path.parent().expect("database path has a parent"), + TempDir::path, + ) } pub fn db_path(&self) -> &Path { @@ -147,11 +173,8 @@ impl RealServer { pub fn shutdown_and_verify(mut self) -> CleanupReport { let child_id = self.child.id(); let port = self.port; - let temp_path = self - .temp_dir - .as_ref() - .map(|dir| dir.path().to_path_buf()) - .expect("server temp dir should be present before shutdown"); + let temp_path = self.temp_path().to_path_buf(); + let owns_temp_dir = self.temp_dir.is_some(); let redacted_stderr = stop_child_and_redact_stderr( &mut self.child, &mut self.stderr_thread, @@ -159,7 +182,7 @@ impl RealServer { &self.api_key, ); drop(self.temp_dir.take()); - let temp_removed = wait_until(CLEANUP_TIMEOUT, || !temp_path.exists()); + let temp_removed = owns_temp_dir && wait_until(CLEANUP_TIMEOUT, || !temp_path.exists()); let port_released = port.map(|p| wait_until(CLEANUP_TIMEOUT, || port_is_bindable(p))); CleanupReport { child_id,