diff --git a/.github/workflows/auto-merge-dependabot.yml b/.github/workflows/auto-merge-dependabot.yml index 45c596c8..810b2e00 100644 --- a/.github/workflows/auto-merge-dependabot.yml +++ b/.github/workflows/auto-merge-dependabot.yml @@ -30,7 +30,7 @@ jobs: steps: - name: Fetch Dependabot metadata id: meta - uses: dependabot/fetch-metadata@v3 + uses: dependabot/fetch-metadata@25dd0e34f4fe68f24cc83900b1fe3fe149efef98 # v3.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d6d55cfc..461f407c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -43,12 +43,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: main - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: # Read from go.mod so the toolchain is declared once. go-version-file: go.mod diff --git a/.github/workflows/build-sandbox-image.yml b/.github/workflows/build-sandbox-image.yml index 71aca0d2..b44ada1c 100644 --- a/.github/workflows/build-sandbox-image.yml +++ b/.github/workflows/build-sandbox-image.yml @@ -98,16 +98,38 @@ jobs: BUNDLE_DIR: ${{ inputs.bundle_dir }} steps: - name: Checkout caller repo - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # The build script lives in fleet core; a client-config caller doesn't # have it. Checking it out into a subdir keeps ONE canonical builder # (same manifest parsing, same flags) instead of a drifting copy per repo. + # fleet_ref chooses which ref of ElcanoTek/fleet the BUILD SCRIPT below is + # taken from — and that script is then EXECUTED. Every ref in this repo is + # collaborator-written except refs/pull/* (fork PRs), so those are exactly + # the refs that would let non-collaborator code run here; refuse them + # before the checkout. The checkout then consumes this step's validated + # output rather than the raw input. (Found via CodeQL + # actions/untrusted-checkout under security-extended; the same pattern in + # publish-sandbox-image.yml was hardened symmetrically even though the + # query's privileged/taint split happened to flag neither variant there.) + - name: Pin fleet_ref to collaborator-controlled refs + id: pin + env: + REQUESTED: ${{ inputs.fleet_ref || 'main' }} + run: | + set -euo pipefail + case "$REQUESTED" in + refs/pull/*|pull/*|-*) + echo "::error::fleet_ref '$REQUESTED' is refused: pull-request refs carry fork-controlled code, and this workflow executes the checked-out build script." + exit 1 ;; + esac + printf 'resolved=%s\n' "$REQUESTED" >> "$GITHUB_OUTPUT" + - name: Checkout fleet (build script) - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ElcanoTek/fleet - ref: ${{ inputs.fleet_ref || 'main' }} + ref: ${{ steps.pin.outputs.resolved }} path: .fleet-core sparse-checkout: | scripts/build-sandbox-image.sh @@ -130,14 +152,26 @@ jobs: - name: Report what was built if: ${{ always() && steps.build.outcome != 'skipped' }} + env: + # The step outcome comes in through env, not interpolated into the + # script: a ${{ }} expression expanded inside a run: block is the + # script-injection shape semgrep's github-actions rules flag, and the + # inline form also broke their bash sub-parser, silently costing this + # file rule coverage (same fix as codeql.yml's $RUNNER_TEMP). + BUILD_OUTCOME: ${{ steps.build.outcome }} run: | tag="$(FLEET_CLIENT_CONFIG_DIR="$GITHUB_WORKSPACE/$BUNDLE_DIR" \ bash .fleet-core/scripts/build-sandbox-image.sh --print-tag 2>/dev/null || true)" - if [ "${{ steps.build.outcome }}" = "success" ]; then + # Plain assignment instead of a ${tag:-(…)} expansion default: the + # parenthesis inside the default value is valid bash but chokes + # semgrep's bash sub-parser, which partial-parsed this file and + # silently dropped two rules' coverage of it. + if [ -z "$tag" ]; then tag="(tag unavailable)"; fi + if [ "$BUILD_OUTCOME" = "success" ]; then { echo "### Sandbox builds clean" echo - echo "\`${tag:-(tag unavailable)}\` built from \`$BUNDLE_DIR/sandbox/Containerfile\`." + echo "\`$tag\` built from \`$BUNDLE_DIR/sandbox/Containerfile\`." echo echo "Nothing was pushed — this is a build canary. The base tracks" echo "\`fedora-minimal:latest\`, so this is the check that a Fedora" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5a8ebdc..99bf8d60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: docs_only: ${{ steps.detect.outputs.docs_only }} steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Full history so the base/head (or before/after) SHAs are present to diff. fetch-depth: 0 @@ -78,7 +78,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install gitleaks # Pin a specific gitleaks release and verify its checksum so the gate is @@ -110,7 +110,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Full history so the merge-base with origin/main resolves and the # linter can diff the branch's new/changed migration files. @@ -163,10 +163,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: # Read from go.mod so the toolchain is declared once. go-version-file: go.mod @@ -231,7 +231,7 @@ jobs: run: go vet -tags fleet_host_executor ./... - name: golangci-lint - uses: golangci/golangci-lint-action@v9 + uses: golangci/golangci-lint-action@db9de0fc1a667e1a49d2291a1a042dff081d78f6 # v9 with: # Repo .golangci.yml is the v2 schema (version: "2"). It no longer # pins run.go: golangci-lint's documented default is "use Go version @@ -316,6 +316,71 @@ jobs: # on a schedule instead of ambushing the next unrelated PR. run: go run golang.org/x/vuln/cmd/govulncheck@latest ./... + python: + name: Python lint (ruff) + runs-on: ubuntu-latest + needs: changes + # Skipped for a docs-only change; the `CI gate` job treats a skip as a pass. + if: ${{ needs.changes.outputs.docs_only != 'true' }} + # The lane the repo did not have. fleet ships 13 Python files — the sandbox + # FileOp helper, the python bridge, the bento-slides and data-profiler skill + # scripts, MCP test servers — and nothing linted any of them: Go had + # golangci-lint, the web tier had oxlint, Python had neither. Its only + # coverage was whatever CodeQL's code-quality suite happened to notice, which + # is a ~40s job with no autofix; ruff does the same class of check in well + # under a second. See ruff.toml for why the rule set is narrow (default rules + # find 3 issues here; the broad selection finds 333, almost all style churn). + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install ruff + # Pinned like every other tool this repo installs in CI, so an upstream + # release cannot change the verdict without a visible diff. + env: + RUFF_VERSION: '0.15.8' + run: | + set -euo pipefail + python3 -m pip install --user --quiet "ruff==${RUFF_VERSION}" + # GITHUB_PATH only affects LATER steps, so export for this one too. + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.local/bin:$PATH" + ruff --version + + - name: Lint + # Config (rule selection, excludes, per-file ignores) lives in ruff.toml + # so a local `ruff check .` and this gate cannot disagree. + run: ruff check --output-format github . + + - name: Formatting check + # A gate, and safe to be one: the whole tree was ruff-formatted in the + # change that flipped this, so a failure here is a NEW unformatted file, + # fixable with one `ruff format .`. + run: ruff format --check . + + codeql: + # Reusable-workflow call: brings codeql.yml's jobs into THIS workflow's + # graph so `CI gate` (the single required check on main) blocks on them. + # Its `Fail on findings` step means a finding — not just a broken scanner — + # fails this job and therefore the gate. + needs: changes + # Docs-only changes cannot touch Go/JS/Python/workflow code (the allowlist + # is *.md, docs/, LICENSE); the gate treats the skip as a pass. + if: ${{ needs.changes.outputs.docs_only != 'true' }} + permissions: + contents: read + security-events: write # the called workflow uploads SARIF to code scanning + actions: read + uses: ./.github/workflows/codeql.yml + + semgrep: + # Same mechanism as codeql above: called here so `CI gate` blocks on it. + needs: changes + if: ${{ needs.changes.outputs.docs_only != 'true' }} + permissions: + contents: read + uses: ./.github/workflows/semgrep.yml + web: name: Web lint / test / build runs-on: ubuntu-latest @@ -328,16 +393,43 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: # Read from web/.nvmrc so the deployed node major is declared once. node-version-file: web/.nvmrc cache: npm cache-dependency-path: web/package-lock.json + - name: Audit dependencies for known CVEs + # The npm counterpart of the Go job's govulncheck gate: reads + # package-lock.json against the npm advisory DB and fails on ANY + # severity. Like govulncheck, the verdict is a function of the clock as + # well as the commit — a new advisory can redden an unchanged tree, and + # that is the point. Needs no node_modules (lockfile-only), so it runs + # before the expensive install and fails fast. + run: npm audit --audit-level=low + + - name: Audit rampart-service dependencies for known CVEs + # Separate tree, same gate. Its lockfile pins `overrides` forcing + # sharp >=0.35 (libvips CVEs) and adm-zip >=0.6 (GHSA-xcpc-8h2w-3j85) + # because no release of @huggingface/transformers/onnxruntime-node has + # picked the fixes up yet — see scripts/rampart-service/package.json. + working-directory: scripts/rampart-service + run: npm audit --audit-level=low + + - name: Check whether the rampart security overrides are droppable + # scripts/rampart-service/package.json force-patches sharp and adm-zip + # because their parents have not released fixes. The day upstream does, + # this FAILS with removal instructions — an override left behind after + # upstream fixes itself silently pins Dependabot's updates down. A + # registry flake skips with a notice (the audit above is the CVE gate). + # Absolute path: this job's default working-directory is web/, which is + # exactly how run 32579378165 caught the repo-relative form (exit 127). + run: "$GITHUB_WORKSPACE/scripts/check-npm-overrides.sh" + - name: Install dependencies run: npm ci @@ -389,10 +481,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: # Read from web/.nvmrc so the deployed node major is declared once. node-version-file: web/.nvmrc @@ -415,7 +507,7 @@ jobs: - name: Upload Playwright HTML report if: ${{ failure() }} - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: playwright-report path: web/playwright-report/ @@ -460,17 +552,17 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: # Read from go.mod so the toolchain is declared once. go-version-file: go.mod check-latest: true - name: Set up Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: # Read from web/.nvmrc so the deployed node major is declared once. node-version-file: web/.nvmrc @@ -588,7 +680,7 @@ jobs: - name: Upload Playwright live report if: ${{ failure() }} - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: playwright-live-report path: web/playwright-report/ @@ -596,7 +688,7 @@ jobs: - name: Upload server logs on failure if: ${{ failure() }} - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-live-server-logs path: | @@ -621,7 +713,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Grype # Pin a specific Grype release and verify its checksum, exactly as the @@ -694,12 +786,12 @@ jobs: - name: Upload Grype SARIF results to the GitHub Security tab # Upload whenever the scan produced a SARIF — including when the policy - # step rejects a fixable CRITICAL RPM — so + # step rejects a fixable CRITICAL/HIGH RPM — so # findings reach GitHub Security → Code scanning with full CVE details, # affected packages, and fix versions rather than only a red CI job. The # hashFiles guard skips this step (instead of erroring "file not found") # when an EARLIER step failed before grype could write the SARIF. - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@4c0873ef8656cb3c50b3f42fb63bc1ade0cfa827 # v4 (4.37.8) if: ${{ !cancelled() && hashFiles('grype-results.sarif') != '' }} with: sarif_file: 'grype-results.sarif' @@ -715,7 +807,7 @@ jobs: # allowed (docs-only), but any failure or cancellation fails the gate. name: CI gate if: ${{ always() }} - needs: [changes, gitleaks, migrations, go, web, playwright, e2e-live, grype-scan] + needs: [changes, gitleaks, migrations, go, python, codeql, semgrep, web, playwright, e2e-live, grype-scan] runs-on: ubuntu-latest steps: - name: Require all upstream jobs to have succeeded or been skipped diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..00651bec --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,327 @@ +# CodeQL — ADVANCED setup. +# +# This file replaces GitHub's zero-config "default setup", which was switched off +# because its Go analysis could not be fixed from anywhere. Default setup +# installed the Go toolchain its extractor was built with (1.26.6) and pinned +# `GOTOOLCHAIN=local`, so on a `go.mod` that requires 1.27 it could neither build +# that toolchain nor download it: +# +# go: go.mod requires go >= 1.27.0 (running go 1.26.6; GOTOOLCHAIN=local) +# Failed to run `go mod tidy -e` in . +# Extraction failed for all discovered Go projects. +# CodeQL job status was configuration error. +# +# Every main-targeting PR had that failure from the Go 1.27 bump (#1240, promoted +# in #1242) onward, so the repo had no CodeQL coverage of its Go code for that +# whole stretch. Default setup exposes no knobs — there is no place to set a Go +# version or an env var — hence advanced setup, where `actions/setup-go` reads +# `go-version-file: go.mod` like every other lane in this repo. +# +# The literal-version rule applies here too: `scripts/check_versions_test.go`'s +# TestWorkflowsDeclareVersionsByFile scans EVERY .github/workflows/*.yml for a +# literal `go-version:`/`node-version:` and fails on it. That assertion is +# directory-wide, so this file was covered by it the moment it was added — no +# change to the test was needed. Do not replace the version-file input with a +# hardcoded 1.27; that is the exact bug class #1240 and #1241 already fixed twice +# for node, and pinning it here would go stale silently the next time go.mod moves. +# +# SCOPE: security queries only. Default setup ran two analyses per event — +# security over go/python/javascript-typescript/actions, and code quality over +# the first three. This file restores the SECURITY half and deliberately drops +# the code-quality half. That is a considered narrowing, not an oversight, and +# the measurements behind it are recorded in docs/CODEQL.md. In short: +# +# - The quality suite was enabled first and measured. It produced 32 findings, +# every one of them note-level, and ZERO security findings across the whole +# tree. So nothing was being caught that mattered at severity. +# - For Go and the web tier it duplicates linters that already run AND ALREADY +# BLOCK via `ci-gate`: golangci-lint (gosec, staticcheck, revive, unparam, +# gocritic, …) and oxlint. `go/useless-assignment-to-field` is squarely +# inside golangci-lint's remit. +# - 28 of the 32 were Python, which was the one real gap — Python had no +# linter at all. That gap is now closed by ruff (see ruff.toml), which finds +# the same class of thing in under a second, with autofix, and blocks. A +# ~40s CodeQL job with no autofix is the wrong instrument for it. +# - 3 of the 32 were false positives on correct code (the idiomatic NaN test +# `value != value`, flagged as comparison-of-identical-expressions). +# +# Note for anyone tempted to re-add it: `analysis-kinds: code-scanning,code-quality` +# does NOT work in a custom workflow. The action logs two ##[error] lines and +# still exits 0, silently analyzing security only: +# +# The `analysis-kinds` input is experimental and for GitHub-internal use +# only. [...] An analysis kind other than `code-scanning` was specified in a +# custom workflow. This is not supported and will become a fatal error in a +# future version of the CodeQL Action. If your intention is to use quality +# queries outside of Code Quality, use the `queries` input with +# `code-quality` instead. +# [...] Specifying multiple values as input is no longer supported. +# +# `queries: code-quality` is the working form. It is simply not wanted here. +# +# Not touched by this file: the three independent `upload-sarif` calls (ci.yml's +# Grype step, govulncheck-scheduled.yml, grype-scheduled.yml). Those push their +# own SARIF to the Security tab and never depended on CodeQL being configured. +# +# Merge gating, in two parts — both now closed: +# +# 1. Does a finding turn the CHECK red? YES. The `Fail on findings` step below +# fails the job on any finding. Without it the analyze step exits 0 whether +# it found nothing or a hundred alerts, so a red check could only ever mean +# "the scanner broke". +# 2. Does a red check BLOCK a merge? YES, through the EXISTING required check. +# This is a REUSABLE workflow (`on: workflow_call`): ci.yml and dev-ci.yml +# each call it as a job, and a job that calls a reusable workflow can sit in +# a gate job's `needs` like any other job — which routes a CodeQL failure +# into `CI gate` / `Dev gate` with no branch-protection change at all. The +# earlier design note saying this required a repo-settings click was wrong: +# `needs` cannot cross workflow FILES, but a workflow_call brings the jobs +# into the caller's file. +# +# They cannot be part of `CI gate`: a job's `needs` cannot reach across workflow +# files. So this file carries its own aggregate `CodeQL gate` job at the bottom, +# for the same reason ci.yml and dev-ci.yml carry theirs — it is the ONE check to +# name in branch protection if CodeQL should ever become blocking, instead of +# four per-language checks that would have to be re-pointed by hand every time +# the matrix changes. Adding it here does not make it required; that is a +# repo-settings decision, deliberately not expressible from this file. +# See docs/CODEQL.md ("Merge gating"). +name: CodeQL + +on: + # No push/pull_request triggers of its own: per-change runs come from ci.yml + # (push/PR on main) and dev-ci.yml (push/PR on dev) calling this workflow, so + # every branch event is covered exactly once and the result feeds the gates. + # Scanning dev PRs at all is the one place this exceeds old default setup, + # which never ran on them (verified: zero CodeQL runs recorded on dev PRs). + workflow_call: + workflow_dispatch: # manual re-run (e.g. after dismissing an alert) + schedule: + # Monday 10:00 UTC, weekly. Offset from the 07:00 canary, the 08:00 daily + # govulncheck and the Monday 09:00 Grype scan, following the same + # don't-contend-for-runners reasoning those files state. + # + # A cron matters for CodeQL specifically because a run's verdict is a + # function of the query pack as well as the commit: new queries ship + # continuously, and without a schedule the only way this repo learns that a + # newly published query flags existing code is that some unrelated PR turns + # red. Same argument govulncheck-scheduled.yml makes about vuln.go.dev. + - cron: '0 10 * * 1' + +permissions: + contents: read + security-events: write # required to upload CodeQL results to the Security tab + # The `actions` analysis reads workflow definitions; `packages: read` is not + # needed because no analysis here pulls a private CodeQL pack. + actions: read + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + include: + # Go is the reason this file exists. `build-mode: none` is NOT + # supported for Go — only `autobuild` or `manual` — so the toolchain + # has to be right rather than skipped. + # security-extended everywhere: the broader security suite (more + # queries, lower average precision than the default). Measured before + # adoption like every other gate here — the run's own findings + # summary is the measurement, and the Fail-on-findings step means + # anything it surfaces must be fixed or reasoned away, not accrued. + - language: go + build-mode: autobuild + queries: security-extended + - language: python + build-mode: none + queries: security-extended + - language: javascript-typescript + build-mode: none + queries: security-extended + - language: actions + build-mode: none + queries: security-extended + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # THE FIX. Runs before init so the interpreter is already on PATH when the + # Go extractor and autobuild shell out to `go`. go-version-file makes go.mod + # the single declaration point, so this cannot drift from the module. + - name: Set up Go + if: matrix.language == 'go' + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + + - name: Initialize CodeQL + uses: github/codeql-action/init@4c0873ef8656cb3c50b3f42fb63bc1ade0cfa827 # v4 (4.37.8) + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + queries: ${{ matrix.queries }} + # security-extended (see the matrix note). The SCOPE note in the + # header still applies to `code-quality` — do not re-add that one. + + - name: Autobuild + if: matrix.build-mode == 'autobuild' + uses: github/codeql-action/autobuild@4c0873ef8656cb3c50b3f42fb63bc1ade0cfa827 # v4 (4.37.8) + env: + # Same build tag ci.yml and dev-ci.yml pass to `go vet` and `go test`, + # and for the same reason: internal/sandbox/host.go — the UNSANDBOXED + # host executor — is fenced behind `//go:build fleet_host_executor`, so + # without the tag it is not in the default build and the extractor never + # sees it. Measured on the first run of this workflow: 426 of the 427 + # non-test .go files were extracted, and the one missing file was + # exactly host.go. Leaving the most security-sensitive file in the tree + # as the single hole in Go coverage is not a defensible default, so the + # tag is passed here to match the lanes that already vet it. + GOFLAGS: -tags=fleet_host_executor + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@4c0873ef8656cb3c50b3f42fb63bc1ade0cfa827 # v4 (4.37.8) + with: + category: /language:${{ matrix.language }} + # Also write the SARIF to disk so the step below can summarize it. The + # results are still uploaded to code scanning (`upload` defaults true); + # this only adds a local copy. + output: ${{ runner.temp }}/codeql-sarif + + - name: Summarize findings to the job log + # WHY THIS EXISTS: a CodeQL run reports nothing about what it FOUND to + # its own log. It writes SARIF, uploads it, and exits 0 — with findings + # or without them. So the only way to see results was to open the + # Security tab, which means a run's outcome is invisible to anyone + # reading CI output, to `gh run view`, and to any automation that has + # the log but not the code-scanning API. + # + # This mirrors what govulncheck-scheduled.yml already does for its own + # SARIF: jq a per-rule count into the job log AND the step summary, so + # the run is readable without leaving Actions. It is reporting only and + # never fails the job — gating on findings is code scanning merge + # protection's job (see docs/CODEQL.md), not this step's. + if: ${{ !cancelled() }} + env: + SARIF_DIR: ${{ runner.temp }}/codeql-sarif + LANGUAGE: ${{ matrix.language }} + # The database directory is named after the EXTRACTOR, which is not + # always the matrix language: javascript-typescript -> javascript. + CODEQL_DB: ${{ matrix.language == 'javascript-typescript' && 'javascript' || matrix.language }} + run: | + set -uo pipefail + shopt -s nullglob + files=("$SARIF_DIR"/*.sarif) + { + echo "### CodeQL findings — $LANGUAGE" + if [ ${#files[@]} -eq 0 ]; then + # Not a failure: the analyze step is what fails on a broken run. + # Say it plainly rather than printing "No findings." for a scan + # that never produced a file — reporting a clean result you did + # not observe is the error this repo keeps writing down. + echo 'No SARIF file was written — nothing to summarize (see the analyze step).' + else + echo '```' + # `-s` slurps every SARIF doc into one array, so a language that + # emits more than one file is still counted once, in total. + jq -rs ' + [ .[] | .runs[]? | .results[]? ] as $res + | if ($res | length) == 0 then "No findings." + else + ( $res + | map( + "[\(.level // "note")] \(.ruleId) " + + ((.locations[0].physicalLocation // {}) as $l + | "\($l.artifactLocation.uri // "?"):\($l.region.startLine // "?")") + ) + | sort | join("\n") + ) + "\n--\ntotal findings: \($res | length)" + end + ' "${files[@]}" + echo '```' + # COVERAGE, not just the verdict: "No findings." alone cannot be + # told apart from "analyzed nothing", which is the exact + # green-but-vacuous outcome this workflow exists to rule out. + # The source archive is the file set the database was built from. + # RUNNER_TEMP (the env var), not ${{ runner.temp }}: interpolating a + # GitHub expression straight into a run: block is the shape + # semgrep's gha-curl-pipe-shell / curl-eval rules flag, and it also + # breaks their bash sub-parser — which silently costs coverage on + # this very file. The env var is equivalent and parses. + src="$RUNNER_TEMP/codeql_databases/$CODEQL_DB/src.zip" + if [ -f "$src" ]; then + echo "files in the $LANGUAGE database: $(unzip -Z1 "$src" 2>/dev/null | grep -vc '/$' || echo '?')" + fi + fi + } | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Fail on findings + # THIS is what makes CodeQL a gate rather than a report. The analyze step + # exits 0 whether it found nothing or a hundred alerts — job success only + # means extraction and query evaluation worked. Without this step a red + # `Analyze (…)` check can only ever mean "the scanner broke", never "the + # code has a problem", which is precisely how the Go toolchain break sat + # unnoticed behind a red-but-not-required check for weeks. + # + # Threshold is ANY finding, deliberately. The security suite currently + # reports ZERO across go/python/javascript-typescript/actions, so there + # is no backlog to grandfather and no severity line to argue about — a + # finding here is new. Switching a gate on over an existing backlog is + # how a gate becomes something people route around. + # + # Runs after the summary so the log leads with WHAT was found. + # + # NOTE ON SCOPE: this makes the CHECK red. Whether a red check BLOCKS a + # merge is branch protection's call — `CodeQL gate` has to be a required + # status check for that, which is a repo-settings action a workflow file + # cannot perform. See docs/CODEQL.md ("Merge gating"). + if: ${{ !cancelled() }} + env: + SARIF_DIR: ${{ runner.temp }}/codeql-sarif + LANGUAGE: ${{ matrix.language }} + run: | + set -uo pipefail + shopt -s nullglob + files=("$SARIF_DIR"/*.sarif) + if [ ${#files[@]} -eq 0 ]; then + # No SARIF means the analysis did not produce results to judge. Fail + # loudly rather than reporting a clean scan that never happened. + echo "::error::No SARIF written for $LANGUAGE — cannot conclude the scan was clean." + exit 1 + fi + count=$(jq -rs '[ .[] | .runs[]? | .results[]? ] | length' "${files[@]}") + if [ "$count" != "0" ]; then + echo "::error::CodeQL found ${count} finding(s) for ${LANGUAGE} — see the summary above." + echo "Fix it, or if it is a false positive dismiss the alert in the" + echo "Security tab with a reason, or add a query filter with a comment" + echo "saying why. Silently raising the threshold is not one of the options." + exit 1 + fi + echo "CodeQL ($LANGUAGE): 0 findings." + + codeql-gate: + name: CodeQL gate + # Aggregate of the matrix. In the workflow_call path the caller's + # `needs: codeql` already rolls up every job here, so this exists for the + # standalone schedule/dispatch runs — one legible verdict per weekly re-scan + # instead of four boxes — and as a stable single check name. + # + # `needs: [analyze]` on a matrix job collapses to one aggregate result: + # success only when every leg succeeded. With `fail-fast: false` above, + # every language still runs and reports before this evaluates them. + if: always() + needs: [analyze] + runs-on: ubuntu-latest + steps: + - name: Fail if any CodeQL analysis did not succeed + run: | + results='${{ join(needs.*.result, ' ') }}' + echo "job results: $results" + for r in $results; do + [ "$r" = "success" ] || { echo "a CodeQL analysis did not succeed"; exit 1; } + done diff --git a/.github/workflows/dev-ci.yml b/.github/workflows/dev-ci.yml index 32831baa..b4153e6f 100644 --- a/.github/workflows/dev-ci.yml +++ b/.github/workflows/dev-ci.yml @@ -11,15 +11,21 @@ # # What runs here: Go compile+vet+lint+test WITH a Postgres service (#723 — so the # DB-gated suites, including the entire `fleet import` suite, are exercised on dev -# instead of first firing at the dev→main promotion; no -race lane), the web -# lint/test/build lane, the migration DDL lint, and the gitleaks secret scan (the -# no-secrets invariant is never weakened, on any branch). +# instead of first firing at the dev→main promotion; no -race lane), the Python +# lint (ruff), the web lint/test/build lane, the migration DDL lint, and the +# gitleaks secret scan (the no-secrets invariant is never weakened, on any +# branch). # # Still deliberately deferred to the dev→main PR's full ci.yml gate, because each # is slow and none of them is what a routine change breaks: the -race lane, -# govulncheck, the Grype image scan, both Playwright suites, and CodeQL. The -# division of labour is "does it compile, lint, and pass tests" here; "is it safe -# to ship" there. +# govulncheck, the Grype image scan, and both Playwright suites. The division of +# labour is "does it compile, lint, and pass tests" here; "is it safe to ship" +# there. +# +# CodeQL and Semgrep used to be on that deferred list and no longer are: both +# run IN this lane, as reusable-workflow calls wired into `Dev gate`, so a +# scanner finding blocks dev the same way a compile error does. See +# docs/SCANNING.md. # # `Dev gate` is the single aggregate job (same pattern as ci.yml's `CI gate`): # when the dev branch is protected, require just that one check. @@ -78,10 +84,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod cache: true @@ -101,7 +107,7 @@ jobs: run: go vet -tags fleet_host_executor ./... - name: golangci-lint - uses: golangci/golangci-lint-action@v9 + uses: golangci/golangci-lint-action@db9de0fc1a667e1a49d2291a1a042dff081d78f6 # v9 with: # Keep pinned in lockstep with ci.yml + .golangci.yml (see the note # there) so the fast lane and the full gate never disagree. @@ -129,6 +135,57 @@ jobs: # compile with it. run: go test -p 1 -tags fleet_host_executor ./... + python: + name: Python lint (ruff) + runs-on: ubuntu-latest + # Mirrors ci.yml's `python` job exactly, for the same reason the web lane was + # added to this file: a change should not first be checked at the dev->main + # promotion. ruff takes about a second, so there is no speed argument for + # deferring it. + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install ruff + env: + # Keep pinned in lockstep with ci.yml (asserted by + # scripts/check_versions_test.go) so the fast lane and the full gate + # cannot disagree. + RUFF_VERSION: '0.15.8' + run: | + set -euo pipefail + python3 -m pip install --user --quiet "ruff==${RUFF_VERSION}" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.local/bin:$PATH" + ruff --version + + - name: Lint + run: ruff check --output-format github . + + - name: Formatting check + # Mirrors ci.yml's gate; the tree is ruff-formatted, so a failure here + # is a new unformatted file. + run: ruff format --check . + + codeql: + # Reusable-workflow call (see codeql.yml's header): puts the CodeQL jobs in + # THIS graph so `Dev gate` blocks on them. A finding fails the gate — the + # `Fail on findings` step inside makes green mean "clean", not just "ran". + # Unconditional (no docs-only detection in the fast lane, and Dev gate + # demands strict success, so a skip would fail it); running on dev pushes + # too also covers any direct push that bypassed a PR. + permissions: + contents: read + security-events: write + actions: read + uses: ./.github/workflows/codeql.yml + + semgrep: + # Same mechanism: called here so `Dev gate` blocks on it. + permissions: + contents: read + uses: ./.github/workflows/semgrep.yml + web: name: Web lint / test / build (fast) runs-on: ubuntu-latest @@ -142,16 +199,35 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: # Read from web/.nvmrc so the deployed node major is declared once. node-version-file: web/.nvmrc cache: npm cache-dependency-path: web/package-lock.json + - name: Audit dependencies for known CVEs + # Mirrors ci.yml's web job: lockfile-only, fails on any severity, runs + # before the install so a vulnerable lockfile fails fast. + run: npm audit --audit-level=low + + - name: Audit rampart-service dependencies for known CVEs + working-directory: scripts/rampart-service + run: npm audit --audit-level=low + + - name: Check whether the rampart security overrides are droppable + # scripts/rampart-service/package.json force-patches sharp and adm-zip + # because their parents have not released fixes. The day upstream does, + # this FAILS with removal instructions — an override left behind after + # upstream fixes itself silently pins Dependabot's updates down. A + # registry flake skips with a notice (the audit above is the CVE gate). + # Absolute path: this job's default working-directory is web/, which is + # exactly how run 32579378165 caught the repo-relative form (exit 127). + run: "$GITHUB_WORKSPACE/scripts/check-npm-overrides.sh" + - name: Install dependencies run: npm ci @@ -180,7 +256,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Full history so the merge-base with origin/main resolves and the # linter can diff the branch's new/changed migration files. @@ -203,7 +279,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install gitleaks # Same pinned release + checksum as ci.yml — reproducible, can't be @@ -229,7 +305,7 @@ jobs: # Aggregate check for branch protection: passes only when every fast-lane # job succeeded (mirrors ci.yml's `CI gate`). if: always() - needs: [go, web, migrations, gitleaks] + needs: [go, python, codeql, semgrep, web, migrations, gitleaks] runs-on: ubuntu-latest steps: - name: Fail if any fast-lane job failed diff --git a/.github/workflows/e2e-canary.yml b/.github/workflows/e2e-canary.yml index 1a6f2d04..aa5aac1c 100644 --- a/.github/workflows/e2e-canary.yml +++ b/.github/workflows/e2e-canary.yml @@ -76,17 +76,17 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: # Read from go.mod so the toolchain is declared once. go-version-file: go.mod check-latest: true - name: Set up Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: # Read from web/.nvmrc so the deployed node major is declared once. node-version-file: web/.nvmrc @@ -130,7 +130,7 @@ jobs: - name: Upload canary report on failure if: ${{ failure() }} - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: canary-report path: | diff --git a/.github/workflows/govulncheck-scheduled.yml b/.github/workflows/govulncheck-scheduled.yml index 83f5c980..0e4a09e1 100644 --- a/.github/workflows/govulncheck-scheduled.yml +++ b/.github/workflows/govulncheck-scheduled.yml @@ -39,6 +39,7 @@ on: permissions: contents: read security-events: write # required to upload SARIF to the Security tab + issues: write # the failure-alarm step files an issue when a scheduled run breaks jobs: govulncheck-scheduled: @@ -47,12 +48,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: main # always scan the tip of main, not a PR branch - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod cache: true @@ -91,10 +92,38 @@ jobs: echo '```' >> "$GITHUB_STEP_SUMMARY" - name: Upload scan results - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@4c0873ef8656cb3c50b3f42fb63bc1ade0cfa827 # v4 (4.37.8) # Skip cleanly (rather than error "file not found") if the scan step # failed before govulncheck could write the SARIF. if: ${{ !cancelled() && hashFiles('govulncheck.sarif') != '' }} with: sarif_file: 'govulncheck.sarif' category: 'govulncheck-scheduled' + + - name: File an issue so a red cron cannot rot silently + # A scheduled failure has no PR attached, so nothing surfaces it — the + # exact rot pattern that let the CodeQL toolchain break sit red for + # weeks. Files one issue per breakage (deduped by title), comments on + # re-failures. schedule-only: a red manual dispatch has a human watching. + # This step body is duplicated across the four scheduled scan lanes + # (codeql, semgrep, govulncheck, grype) — keep them in sync. + if: ${{ failure() && github.event_name == 'schedule' }} + env: + GH_TOKEN: ${{ github.token }} + WORKFLOW_NAME: ${{ github.workflow }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + title="Scheduled ${WORKFLOW_NAME} run failed" + num="$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \ + --search "\"$title\" in:title" --json number --jq '.[0].number // empty' || true)" + if [ -n "$num" ]; then + gh issue comment "$num" --repo "$GITHUB_REPOSITORY" \ + --body "Still failing: $RUN_URL" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" \ + --body "The scheduled $WORKFLOW_NAME lane went red: $RUN_URL + + A failing cron has no PR to make it visible, so this issue is the + alarm. Close it when the lane is green again." + fi diff --git a/.github/workflows/grype-scheduled.yml b/.github/workflows/grype-scheduled.yml index 20e9b09f..b2357c52 100644 --- a/.github/workflows/grype-scheduled.yml +++ b/.github/workflows/grype-scheduled.yml @@ -22,6 +22,7 @@ on: permissions: contents: read security-events: write # required to upload SARIF to the Security tab + issues: write # the failure-alarm step files an issue when a scheduled run breaks jobs: grype-scheduled: @@ -32,7 +33,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: main # always scan the tip of main, not a PR branch @@ -79,10 +80,38 @@ jobs: --output sarif=grype-results.sarif - name: Upload weekly scan results - uses: github/codeql-action/upload-sarif@v4 + uses: github/codeql-action/upload-sarif@4c0873ef8656cb3c50b3f42fb63bc1ade0cfa827 # v4 (4.37.8) # Skip cleanly (rather than error "file not found") if an earlier step # failed before grype could write the SARIF. if: ${{ !cancelled() && hashFiles('grype-results.sarif') != '' }} with: sarif_file: 'grype-results.sarif' category: 'grype-scheduled' + + - name: File an issue so a red cron cannot rot silently + # A scheduled failure has no PR attached, so nothing surfaces it — the + # exact rot pattern that let the CodeQL toolchain break sit red for + # weeks. Files one issue per breakage (deduped by title), comments on + # re-failures. schedule-only: a red manual dispatch has a human watching. + # This step body is duplicated across the four scheduled scan lanes + # (codeql, semgrep, govulncheck, grype) — keep them in sync. + if: ${{ failure() && github.event_name == 'schedule' }} + env: + GH_TOKEN: ${{ github.token }} + WORKFLOW_NAME: ${{ github.workflow }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + title="Scheduled ${WORKFLOW_NAME} run failed" + num="$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \ + --search "\"$title\" in:title" --json number --jq '.[0].number // empty' || true)" + if [ -n "$num" ]; then + gh issue comment "$num" --repo "$GITHUB_REPOSITORY" \ + --body "Still failing: $RUN_URL" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" \ + --body "The scheduled $WORKFLOW_NAME lane went red: $RUN_URL + + A failing cron has no PR to make it visible, so this issue is the + alarm. Close it when the lane is green again." + fi diff --git a/.github/workflows/publish-sandbox-image.yml b/.github/workflows/publish-sandbox-image.yml index 50393990..8be79186 100644 --- a/.github/workflows/publish-sandbox-image.yml +++ b/.github/workflows/publish-sandbox-image.yml @@ -215,16 +215,38 @@ jobs: SHA: ${{ github.sha }} steps: - name: Checkout caller repo - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # The build script lives in fleet core; a client-config caller doesn't # have it. Checking it out into a subdir keeps ONE canonical builder # (same manifest parsing, same flags) instead of a drifting copy per repo. + # fleet_ref chooses which ref of ElcanoTek/fleet the BUILD SCRIPT below is + # taken from — and that script is then EXECUTED. Every ref in this repo is + # collaborator-written except refs/pull/* (fork PRs), so those are exactly + # the refs that would let non-collaborator code run here; refuse them + # before the checkout. The checkout then consumes this step's validated + # output rather than the raw input. (Found via CodeQL + # actions/untrusted-checkout under security-extended; the same pattern in + # build-sandbox-image.yml was flagged; this file escaped both query + # variants despite holding packages: write — the MORE dangerous twin.) + - name: Pin fleet_ref to collaborator-controlled refs + id: pin + env: + REQUESTED: ${{ inputs.fleet_ref || 'main' }} + run: | + set -euo pipefail + case "$REQUESTED" in + refs/pull/*|pull/*|-*) + echo "::error::fleet_ref '$REQUESTED' is refused: pull-request refs carry fork-controlled code, and this workflow executes the checked-out build script." + exit 1 ;; + esac + printf 'resolved=%s\n' "$REQUESTED" >> "$GITHUB_OUTPUT" + - name: Checkout fleet (build script) - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ElcanoTek/fleet - ref: ${{ inputs.fleet_ref || 'main' }} + ref: ${{ steps.pin.outputs.resolved }} path: .fleet-core sparse-checkout: | scripts/build-sandbox-image.sh diff --git a/.github/workflows/scan-cron-alarm.yml b/.github/workflows/scan-cron-alarm.yml new file mode 100644 index 00000000..ac12b94a --- /dev/null +++ b/.github/workflows/scan-cron-alarm.yml @@ -0,0 +1,65 @@ +# Scan cron alarm — files an issue when a SCHEDULED CodeQL or Semgrep run fails. +# +# WHY THIS IS ITS OWN WORKFLOW rather than a job inside codeql.yml/semgrep.yml: +# those two are REUSABLE workflows called by ci.yml and dev-ci.yml, and a called +# workflow may not request token permissions its caller did not grant — the +# check happens at PLAN time, before any `if:` can skip the job. Verified the +# hard way: adding an `issues: write` alarm job inside them failed the entire +# calling Dev CI run with `startup_failure` (run 32578976517), which meant NO +# scanning ran on that head at all. A `workflow_run` watcher has no caller, so +# it can hold `issues: write` without widening any gate's token. +# +# govulncheck-scheduled.yml and grype-scheduled.yml keep their in-job alarm +# steps: they are standalone workflows with no callers, so the constraint above +# does not apply to them. +# +# Why an alarm exists at all: a scheduled failure has no PR attached, so nothing +# surfaces it — the exact rot pattern that let the CodeQL toolchain break sit +# red for weeks. One issue per breakage (deduped by title); re-failures comment +# on the same issue. +# +# workflow_run only fires from this file's copy on the DEFAULT branch, so the +# alarm arms once this merges to main — which is also when the crons themselves +# start mattering. +name: Scan cron alarm + +on: + workflow_run: + workflows: [CodeQL, Semgrep] + types: [completed] + +permissions: + issues: write + +jobs: + alarm: + name: File an issue so a red cron cannot rot silently + # Scheduled failures only: a red workflow_call run already reddens the + # calling gate on a PR, and a red manual dispatch has a human watching it. + if: >- + github.event.workflow_run.conclusion == 'failure' && + github.event.workflow_run.event == 'schedule' + runs-on: ubuntu-latest + steps: + - name: File or update the alarm issue + # Body mirrors the in-job alarm steps in govulncheck-scheduled.yml and + # grype-scheduled.yml — keep the three in sync. + env: + GH_TOKEN: ${{ github.token }} + WORKFLOW_NAME: ${{ github.event.workflow_run.name }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + run: | + set -euo pipefail + title="Scheduled ${WORKFLOW_NAME} run failed" + num="$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \ + --search "\"$title\" in:title" --json number --jq '.[0].number // empty' || true)" + if [ -n "$num" ]; then + gh issue comment "$num" --repo "$GITHUB_REPOSITORY" \ + --body "Still failing: $RUN_URL" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$title" \ + --body "The scheduled $WORKFLOW_NAME lane went red: $RUN_URL + + A failing cron has no PR to make it visible, so this issue is the + alarm. Close it when the lane is green again." + fi diff --git a/.github/workflows/screenshots.yml b/.github/workflows/screenshots.yml index a5647257..7eef6e50 100644 --- a/.github/workflows/screenshots.yml +++ b/.github/workflows/screenshots.yml @@ -50,10 +50,10 @@ jobs: ORCHESTRATOR_SERVER_URL: http://127.0.0.1:18000 steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: # Read from web/.nvmrc so the deployed node major is declared once. node-version-file: web/.nvmrc @@ -75,7 +75,7 @@ jobs: run: npm run test:e2e:screenshots || echo "::warning::GUI screenshot capture failed; keeping the existing images" - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: # Read from go.mod so the toolchain is declared once. go-version-file: go.mod diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml new file mode 100644 index 00000000..50c0a8f5 --- /dev/null +++ b/.github/workflows/semgrep.yml @@ -0,0 +1,215 @@ +# Semgrep — the fast, log-readable scan lane. +# +# WHY A SECOND SCANNER AT ALL. CodeQL is the deep one: interprocedural taint, +# which is what actually checks this project's headline invariants (a credential +# must not reach a log sink, the model context, or the sandbox). It is also slow +# to build a database and reports nothing about what it found to its own log. +# Semgrep is the opposite trade — seconds, no build, findings straight to stdout, +# and rules that are cheap to write. It complements CodeQL; it does not replace +# it, and it must not be treated as a substitute for the taint analysis. +# +# RULESET SCOPE IS EVIDENCE-BASED. The broad registry packs (p/golang, +# p/javascript, p/python) plus p/github-actions were run over this tree and every +# finding was triaged by hand. Result: 51 real findings and 6 false positives. +# +# The 51 were all one rule — `github-actions-mutable-action-tag`: actions pinned +# to a MUTABLE tag (`actions/checkout@v7`) rather than an immutable commit SHA. A +# moved tag runs attacker-controlled code with this repo's token. Every one is +# now FIXED: all 53 action references across every workflow are pinned to a +# 40-hex commit SHA with the version in a trailing comment (which is also the +# form Dependabot updates). +# +# The 6 false positives are suppressed at the line with a `nosemgrep:` comment +# naming the specific rule and the reason. They are worth knowing about, because +# three of them were ALREADY formally triaged and suppressed for gosec (which +# runs inside golangci-lint and already blocks), and one of them is actively +# wrong: +# +# open-redirect cmd/fleet/tls.go +# HTTP->HTTPS upgrade to the SAME host. Already //nolint:gosec G710. +# math-random-used internal/runner/runner.go +# math/rand/v2, used once, for +/-10% jitter on a retry interval. +# cookie-missing-secure internal/sched/handlers/elcano.go +# A DELETION cookie (Value="", MaxAge=-1), no secret. Already //nolint G124. +# unsafe-deserialization-interface internal/mcp/httptool.go +# json.Unmarshal into interface{} is REQUIRED — the value feeds a jq program +# over arbitrary JSON. A concrete struct is not expressible. +# x-frame-options-misconfiguration web/src/proxy.ts +# The header value is the literal string "DENY". No user input reaches it. +# insecure-file-permissions internal/sandbox/fileops.py +# Advises 0o644 — WORLD-READABLE — for a sandbox directory. Following it +# would be a security REGRESSION. 0750 is the file-tool contract. +# +# Each suppression was mutation-tested: removing it makes the finding reappear, +# so a green scan means the waivers are doing the work rather than the rules +# having silently stopped matching. +# +# THIS LANE BLOCKS. `--error` makes semgrep exit non-zero on any finding, and +# there is no `continue-on-error`, so a NEW finding fails the job. That is only +# defensible because the tree is currently at ZERO unsuppressed findings across +# all four packs — verified locally before this was turned on. A gate switched on +# over an unfixed backlog is a gate people learn to ignore. +# +# Adding a suppression is therefore a reviewable act: it shows up in the diff +# next to a reason, which is the property this whole lane exists to have. +# +# RULE PINNING — investigated and REJECTED, on license grounds, not neglect. +# The packs are fetched from the registry at scan time, so a registry-side rule +# addition can turn this gate red with no commit to blame (named in +# docs/SCANNING.md as the first suspect for a mystery red run). Vendoring the +# rule files would fix that, but the Semgrep Rules License v1.0 +# (https://semgrep.dev/legal/rules-license) grants use for "your own internal +# business purposes" only and states outright: "This license does not allow you +# to distribute the rules". Committing them into this public MIT repo would be +# redistribution. The semgrep BINARY version is pinned; the rules deliberately +# are not, with the failure mode documented instead of hidden. +name: Semgrep + +on: + # No push/pull_request triggers of its own: this is a REUSABLE workflow. + # ci.yml and dev-ci.yml call it as a job, which puts it inside `CI gate` / + # `Dev gate` via `needs` — a finding blocks the merge through the existing + # required check, with no branch-protection change. Only the weekly re-scan + # and a manual trigger live here. + workflow_call: + workflow_dispatch: + schedule: + # Monday 11:00 UTC — one hour after CodeQL's weekly, and clear of the 07:00 + # canary / 08:00 govulncheck / 09:00 Grype lanes, following the same + # don't-contend-for-runners note those files carry. Rules ship continuously, + # so like CodeQL this is worth re-running against unchanged code. + - cron: '0 11 * * 1' + +permissions: + contents: read + +jobs: + semgrep: + name: Semgrep scan + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install semgrep + # No actions/setup-python step on purpose: ubuntu-latest already ships a + # python3, and this repo pins no setup-python version anywhere else, so + # adding one would be inventing an unverified action pin for nothing. + # + # The semgrep version IS pinned, like every other tool this repo installs + # in CI (gitleaks, grype, golangci-lint), so an upstream release cannot + # change the findings under us without a visible diff. + env: + SEMGREP_VERSION: '1.174.0' + run: | + set -euo pipefail + python3 -m pip install --user --quiet "semgrep==${SEMGREP_VERSION}" + # GITHUB_PATH only affects LATER steps, so export for this one too. + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.local/bin:$PATH" + semgrep --version + + - name: Scan + run: | + set -uo pipefail + # --metrics=off: never phone home from CI. + # --error: exit non-zero on any finding. THIS is what makes the lane a + # gate. The tree is at zero unsuppressed findings, so a failure here + # means a new one arrived. + # The JSON is written first and the exit status captured, so the + # summary step below still runs and still says WHAT failed — a gate + # that fails without printing the finding is a gate nobody can act on. + semgrep scan \ + --config p/github-actions \ + --config p/golang \ + --config p/javascript \ + --config p/python \ + --metrics=off --error --json -o semgrep.json --quiet + status=$? + if [ ! -s semgrep.json ]; then + echo "semgrep produced no JSON — treating as a scan failure" >&2 + exit 1 + fi + echo "semgrep exit status: $status" + # Defer the failure to the gate step so the summary prints first. + echo "$status" > semgrep.status + + - name: Summarize findings to the job log + # Same reasoning as codeql.yml's summary step: a findings report that + # only exists behind a web UI is unreadable to `gh run view` and to any + # agent holding the log. Print it. + if: ${{ !cancelled() }} + run: | + set -uo pipefail + { + echo '### Semgrep findings' + echo '```' + jq -r ' + (.results // []) as $r + | if ($r | length) == 0 then "No findings." + else + ( $r + | group_by(.check_id) + | sort_by(-length) + | map("\(length | tostring) [\(.[0].extra.severity // "INFO")] \(.[0].check_id | split(".") | last)") + | join("\n") + ) + "\n--\ntotal findings: \($r | length)" + end + ' semgrep.json + echo '```' + # COVERAGE, not just the verdict. "No findings." on its own is + # indistinguishable from "scanned nothing" — the exact + # green-but-vacuous failure this whole stack exists to rule out. So + # print what was actually looked at, and the per-language breakdown + # that shows every pack really applied. + echo '' + echo "files scanned: $(jq '(.paths.scanned // []) | length' semgrep.json)" + echo "files skipped: $(jq '(.paths.skipped // []) | length' semgrep.json)" + echo 'by extension:' + jq -r ' + (.paths.scanned // []) + | map(split(".") | last) + | group_by(.) | map({e: .[0], n: length}) + | sort_by(-.n) | .[:8][] + | " .\(.e): \(.n)" + ' semgrep.json + # Scan errors are not findings but they do mean a rule or file did + # not fully parse, so they get their own line rather than vanishing. + errs=$(jq '(.errors // []) | length' semgrep.json) + if [ "$errs" != "0" ]; then + echo '' + echo "parse/scan errors (a rule or file that did not fully run): $errs" + jq -r '(.errors // [])[] | " [\(.level)] \(.path // "?")"' semgrep.json | sort -u + fi + } | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Fail on findings + # Separate from the scan step purely for ordering: the summary above has + # already printed the per-rule breakdown by the time this fails, so the + # job log leads with WHAT is wrong instead of just that something is. + if: ${{ !cancelled() }} + run: | + set -uo pipefail + status=$(cat semgrep.status 2>/dev/null || echo 1) + count=$(jq '(.results // []) | length' semgrep.json 2>/dev/null || echo '?') + if [ "$status" != "0" ]; then + echo "::error::Semgrep found ${count} unsuppressed finding(s) — see the summary above." + echo "Fix the finding, or, if it is a false positive, add a line-level" + echo "\`nosemgrep: \` comment stating WHY. Both are reviewable in the diff." + exit 1 + fi + echo "Semgrep: 0 unsuppressed findings." + + - name: Upload findings as an artifact + # The repo is public and the results are not sensitive, so the raw JSON + # is kept for a fixing agent (or a human) to consume without re-running + # the scan. + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: semgrep-findings + path: semgrep.json + if-no-files-found: warn + retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index 6ae8fdf1..5466eec8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ make compile # go build ./... (compile-check only; no artifacts) make test # go test -p 1 ./... — run in the FOREGROUND make test-race # go test -race -p 1 ./... (use when touching concurrency) make test-cover # run Go tests with coverage profiling (writes coverage.out) -make lint # golangci-lint + migration DDL lint — must pass clean +make lint # golangci-lint + ruff check/format (Python) + migration DDL lint — must pass clean make fmt # gofmt -w . make tidy # go mod tidy ``` @@ -34,18 +34,26 @@ make tidy # go mod tidy When you touch `web/` (the Next.js app): ```sh -cd web && npm ci && npm run lint && npm run typecheck && npm run test && npm run build +cd web && npm audit --audit-level=low && npm ci && npm run lint && npm run typecheck && npm run test && npm run build cd web && npx playwright test --project=mocked # mocked e2e ``` CI mirrors all of this — Go build/vet/lint/test (including a `-race` lane) plus a `govulncheck` dependency-CVE scan, a Grype container-image CVE scan (fail on a -fixable CRITICAL) of the sandbox image, web lint (oxlint) / typecheck (TS 7) / test / build, Playwright (mocked +fixable CRITICAL/HIGH) of the sandbox image, a Python lint (ruff), web lint (oxlint) / typecheck (TS 7) / test / build, Playwright (mocked **and** live, against a real backend + sandbox), a migration DDL lint, and a gitleaks secret scan. **Every job must be green before merge.** Tests are deterministic without a live model: use the fake-LLM seam (`internal/fakellm` via `OPENROUTER_BASE_URL`), never a real key. +CodeQL (security queries) and Semgrep (Go/JS/Python SAST + Actions supply chain) +also run per PR, **fail on any finding**, and are **inside `ci-gate` and +`Dev gate`** — both are reusable workflows that ci.yml/dev-ci.yml call as jobs, +so a finding blocks the merge through the existing required check. `npm audit` +gates the web and rampart-service dependency trees the same way. Everything is +at zero findings today; keeping it there is the point. See +[`docs/SCANNING.md`](docs/SCANNING.md). + ## Repository map See the README "Repository layout" for the annotated tree. In short: `cmd/` (the @@ -161,6 +169,14 @@ same PR. - **Contributor workflow + CI gates:** [`CONTRIBUTING.md`](CONTRIBUTING.md) - **Testing strategy** (unit / fake-LLM / mocked + live Playwright / canary): [`docs/TESTING.md`](docs/TESTING.md) +- **The scanning stack** (who checks what, why ruff owns Python lint, why + Semgrep is scoped to Actions supply chain after its broad packs scored 0/6, + what blocks vs what reports, and the known gaps): + [`docs/SCANNING.md`](docs/SCANNING.md) +- **CodeQL** (why default setup was replaced by an advanced-setup workflow, how + the Go toolchain is resolved, why it runs security queries only, and the + difference between a required status check and code scanning merge protection): + [`docs/CODEQL.md`](docs/CODEQL.md) - **HTTP API versioning** (the `/v1` prefix + `X-Fleet-API-Version` + `/api-info` discovery + deprecation contract): [`docs/api-versioning.md`](docs/api-versioning.md) - **Database migrations** (the two runners, safe-DDL patterns, the migration DDL diff --git a/CHANGELOG.md b/CHANGELOG.md index 4734038c..61ac4943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,241 @@ prior versions are listed because none have shipped. ### Fixed +- **Every remaining scanner follow-up closed: rule families widened and fixed, + CodeQL at `security-extended`, grype tightened, override canary, cron alarms, + and one reasoned rejection.** + + - **ruff `B`/`SIM`/`S` (bandit) enabled after fixing all 21 measured + findings.** Both `zip()` sites got `strict=True` — each provably + equal-length (one appends to both lists in lockstep; the other sits behind + an explicit `len(row) != len(cols)` guard) — so a future desync fails loud + instead of silently truncating. The unclosed `NamedTemporaryFile` in + bento_doc.py moved inside its `with`. Fourteen deliberate best-effort + `try/except-pass` sites (kernel cleanup, duck-typed pandas/numpy probes, + unlink-on-failure paths) became explicit `contextlib.suppress` with the + intent stated at each. The one `subprocess.Popen` carries a reasoned + `# noqa: S603` (argv is `sys.executable` plus internal literals; + mutation-tested — stripping the noqa re-fires the rule). Full Go suite + green on the result; the sandbox fileops and bridge behavior is covered by + its tests. + + - **CodeQL widened to the `security-extended` suite** on all four languages, + adopted the same way everything else was: the default suite measured zero, + so the broader set starts from a clean baseline and its findings on this + PR's own run are the measurement. That measurement found exactly one thing + — and it was real: `actions/untrusted-checkout/medium` on + `build-sandbox-image.yml`'s `fleet_ref`-fed checkout. Fixed, not waived + (the `actions` language has no `AlertSuppression.ql`, so a comment waiver + does not even exist): the workflow now **refuses `refs/pull/*` refs before + checkout** — a fork-PR ref would put fork-controlled code into a workflow + that executes the checked-out build script — and the identical hardening + went into `publish-sandbox-image.yml`, the *unflagged* twin that holds + `packages: write` and escaped the name-heuristic query only because its + ref plumbing was named differently. Extended suite then verified clean in + CI on all four languages (Dev CI run 525). The CodeQL/Semgrep log summaries + also now print **`file:line` per finding** (plus a database file count as + the coverage line), and the override canary is invoked via + `$GITHUB_WORKSPACE` so it survives the job's `working-directory: web`. + + - **Grype gate tightened to fixable CRITICAL + HIGH**, after measuring: the + published sandbox image carries zero fixable Critical/High RPM findings + (its only fixable findings are two Medium openssh advisories, which the + next routine image rebuild picks up). Policy change mutation-tested in + three directions: real scan passes, injected fixable High fails, injected + fixable Medium still passes. + + - **`scripts/check-npm-overrides.sh`**: the rampart sharp/adm-zip overrides + are forks of upstream's intent, correct only while upstream is broken — so + both CI lanes now fail with removal instructions the day + `@huggingface/transformers` / `onnxruntime-node` publish ranges reaching + the patched lines. Registry flake = skip with a notice, never a verdict. + Mutation-tested in both directions. + + - **A red scheduled scan files an issue** (all four lanes; deduped by title, + re-failures comment). A cron failure has no PR to surface it — the rot + pattern that let the CodeQL toolchain break sit red for weeks. For the two + reusable workflows the alarm lives in `scan-cron-alarm.yml`, a + `workflow_run` watcher, because a called workflow may not request + permissions its caller did not grant — the check fires at PLAN time, before + any `if:` can skip the job, and the first attempt (an `issues: write` job + inside codeql.yml/semgrep.yml) startup-failed the entire calling Dev CI + run. Verified fixed on the next run. + + - **Semgrep rule vendoring investigated and rejected on license grounds**: + the Semgrep Rules License v1.0 permits internal use only and states "This + license does not allow you to distribute the rules" — committing them to + this public MIT repo would be redistribution. The binary stays pinned; the + rules stay registry-fetched with the failure mode documented. + +- **The scanners gate through `ci-gate`/`Dev gate` themselves, npm dependencies + are audited, and the whole Python tree is ruff-formatted — with every finding + fixed, none deferred.** + + - **Gate wiring, corrected.** The previous entry said making CodeQL/Semgrep + merge-blocking needed a branch-protection click, reasoning from "`needs` + cannot cross workflow files". Incomplete: `codeql.yml` and `semgrep.yml` are + now **reusable workflows** (`on: workflow_call`) that ci.yml and dev-ci.yml + call as jobs, and those jobs sit in `ci-gate`'s / `Dev gate`'s `needs` — so a + scanner finding blocks a merge through the one existing required check, no + settings change anywhere. Their own push/pull_request triggers are removed + (nothing runs twice); the weekly re-scan crons and a workflow_dispatch stay. + + - **`npm audit` is a new blocking gate** for both npm trees, lockfile-only and + failing on any severity — the npm counterpart of the govulncheck gate. + `web/` was already clean. `scripts/rampart-service` **had no lockfile at + all**, and generating one exposed **5 high-severity vulnerabilities** it had + been hiding: `sharp <0.35.0` (four libvips CVEs) and `adm-zip <0.6.0` + (GHSA-xcpc-8h2w-3j85) via `onnxruntime-node`. No upstream release fixes + either — latest `@huggingface/transformers` still pins `sharp ^0.34.5`, and + npm's suggested "fix" was a breaking transformers downgrade — so the + package now carries `overrides` to `sharp ^0.35.3` and `adm-zip ^0.6.0`, + each the release immediately after the vulnerable line. The overridden + stack was installed and load-tested, not just resolved: sharp renders a PNG + through the new libvips, transformers loads on it, rampart exports its API, + adm-zip round-trips a zip. Both trees now audit at 0. + + - **`ruff format` applied and gated.** 9 of 13 Python files reformatted + (~3.7k lines), `ruff format --check` now blocks in both CI lanes and in + `make lint`. Validated by the full Go suite (the bento/fileops golden tests + exercise the reformatted scripts), byte-compilation of every file, and a + re-scan showing the fileops `nosemgrep` waiver survived the reformat. + + - **All three semgrep parse errors fixed**, so no file is partially covered: + `${{ steps.build.outcome }}` interpolated into a `run:` script in + build-sandbox-image.yml (moved to `env:` — also the injection-safe form), a + `${tag:-(…)}` expansion default whose bare paren choked the bash sub-parser + (hoisted to a plain assignment), and an inline `import("@playwright/test")` + type in fixtures.ts (now a named `import type`; web lint, tsc and all 1104 + vitest tests pass on it). The scanners' coverage lines now read + **0 parse/scan errors** alongside 0 findings. + +- **The scanners now block, and the repo passes them.** Turning a gate on over an + unfixed backlog is how a gate becomes something people route around, so + everything they reported was fixed or adjudicated first. + + - **All 53 action references pinned to commit SHAs.** Semgrep's + `github-actions-mutable-action-tag` found 51 instances of actions referenced + by a mutable tag (`actions/checkout@v7`); if a tag moves, + attacker-controlled code runs with this repo's `GITHUB_TOKEN`. Every `uses:` + across all 12 workflows is now `@<40-hex-sha> # ` — the form + Dependabot updates, and `.github/dependabot.yml` already watches the + `github-actions` ecosystem. Each SHA is the commit the previously-used tag + resolved to at pin time, so the pin does not smuggle in a version bump. + + - **Semgrep blocks over all four packs** (`p/github-actions`, `p/golang`, + `p/javascript`, `p/python`) with `--error` and no `continue-on-error`. The 6 + false positives are suppressed at the line with `nosemgrep: ` plus a + reason — three of them were *already* triaged and suppressed for gosec, and + one (`0o644` for a sandbox directory) would have been a security regression + if followed. Every suppression was mutation-tested: removing it makes the + finding reappear, so a green scan means the waivers work rather than the + rules having silently stopped matching. + + - **CodeQL fails on findings.** Previously the analyze step exited 0 whether it + found nothing or a hundred alerts, so a red check could only ever mean "the + scanner broke" — which is exactly how the Go toolchain break hid for weeks. + Threshold is any finding, safe because the security suite reports zero across + all four languages. + + Both scanners report as their own checks (`CodeQL gate`, `Semgrep scan`) rather + than through `ci-gate`, because a job's `needs` cannot reach across workflow + files. **Making a red check actually block a merge still requires adding those + two checks to the branch ruleset** — a workflow file cannot make itself + required. + + Two knock-on fixes found while doing this: SHA pinning broke two regexes in + `scripts/check_versions_test.go` that matched `golangci-lint-action@v\d+`, and + they fail *open* by skipping — so they were widened to tolerate a pinned ref + plus its trailing version comment, and mutation-tested to confirm they still + bite. And a standalone `nosemgrep` comment inside a Go import block breaks + `goimports`, so that one waiver is a trailing comment instead. + +- **Python had no linter, and two scanners were pointed at ground already + covered.** Reshaped the scanning stack so each tool owns one job + ([`docs/SCANNING.md`](docs/SCANNING.md)): + + - **ruff is new, and it blocks.** fleet ships 13 Python files — the sandbox + FileOp helper, the python bridge, the bento-slides and data-profiler skill + scripts, MCP test servers — and *nothing* linted any of them. Go had + golangci-lint, the web tier had oxlint, Python had neither. Rule selection is + narrow on purpose and `ruff.toml` records why: the default rules find 3 + findings on this tree, a broad selection finds 333, of which 176 are + `%`-format style and 43 are magic values. Three real findings were fixed to + make the gate clean on day one — an unused import, a lambda assignment, and a + **byte-identical duplicate `has_guard` definition** in `bento_doc.py` where + the second copy silently shadowed the first. `ruff format` is reported but + not gated (the tree has never been ruff-formatted). + + - **CodeQL narrowed to security queries only.** Its code-quality suite was + enabled, measured, and dropped: 32 findings, every one note-level, zero + security findings. For Go and the web tier it duplicated golangci-lint and + oxlint, which already block; 28 of the 32 were Python, now ruff's job; and 3 + were false positives on correct code (`value != value`, the idiomatic NaN + test). CodeQL keeps the thing nothing else here can do — interprocedural + taint, which is the actual shape of "a credential must not reach a log sink". + + - **Semgrep is new, scoped, and advisory.** The obvious move — point it at + `p/golang`/`p/javascript`/`p/python` — was measured and rejected: 6 of 6 + non-Actions findings were false positives, three of them *already* triaged + and suppressed for gosec, and one (`0o644` for a sandbox directory) would + have been a security regression if followed. What ships is + `p/github-actions`, which found 51 instances of one real issue nothing else + checks: actions pinned to mutable tags rather than commit SHAs. Advisory + because all 51 are real and repinning is its own PR, not because they are + doubted. + + - **Both scanners now print findings to the job log** and the step summary, and + Semgrep uploads raw JSON as an artifact. A CodeQL run otherwise reports + nothing about what it found to its own log — it writes SARIF, uploads it, and + exits 0 either way — which made outcomes invisible to `gh run view` and to + any agent holding the log but not the code-scanning API. + + Also added: an aggregate `CodeQL gate` job, so making CodeQL blocking later is + one required check rather than four per-language checks needing manual + re-pointing whenever the matrix changes. Nothing here is wired into `ci-gate` + beyond ruff; CodeQL and Semgrep stay advisory. + +- **CodeQL had stopped analyzing the repo's Go code, and then stopped analyzing + anything.** Default setup's Go analysis failed on every main-targeting PR from + the Go 1.27 bump (#1240, promoted in #1242) onward — it installed the Go its + extractor was built with and pinned `GOTOOLCHAIN=local`, so against a `go.mod` + requiring 1.27 it could neither use nor fetch a workable toolchain: + + ``` + go: go.mod requires go >= 1.27.0 (running go 1.26.6; GOTOOLCHAIN=local) + Extraction failed for all discovered Go projects. + CodeQL job status was configuration error. + ``` + + The failure was red but never blocking (`ci-gate` is the only required check on + main), so it was annotated in promote commit messages and lived on. Default + setup is zero-config, with no Go version input and no env, so there was nothing + to fix in place; switching it off to replace it left the repo scanning nothing + at all in the interim. + + Replaced with an advanced-setup workflow, `.github/workflows/codeql.yml`, which + restores security analysis over go, python, javascript-typescript and actions + plus the code-quality query suite over the first three, and resolves Go's + interpreter from `go.mod` via `actions/setup-go` — never a literal version, the + bug class #1240 and #1241 already fixed twice for node. + + Two things the first cut got wrong, both of which ran **green**: `analysis-kinds` + turns out to be GitHub-internal and unusable in a custom workflow (it logged + `##[error]` and silently continued with security only), and Go extraction + missed exactly one file — `internal/sandbox/host.go`, the unsandboxed host + executor, invisible to the default build behind `//go:build + fleet_host_executor`. Fixed with `queries: code-quality` and + `GOFLAGS: -tags=fleet_host_executor`, the same tag `ci.yml` and `dev-ci.yml` + already pass to `go vet` and `go test`. + + Verified from the extractor's own output rather than the check mark: + `extraction succeeded for all 2 discovered project(s)`, 916 packages, 426 `.go` + files including `host.go`, and distinct queries evaluated rising from 72→116 + (go), 90→292 (python) and 178→374 (javascript-typescript) as the quality suite + came in, with `actions` unchanged at 36 by design. CodeQL remains advisory — + these jobs are deliberately not wired into `ci-gate`. See + [`docs/CODEQL.md`](docs/CODEQL.md). + - **`fleet update` built the web tier on the node it had just refused.** Every update on a Fedora box printed `✓ web tier will build+run on /usr/bin/node-24 (v24.x)` and then, a few lines later, npm's own rejection of that claim: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c89ee51..8bb50ef1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,7 +103,7 @@ Every pull request must be green before merge. CI runs: - **Playwright** — the mocked suite, plus a live suite against a real backend with a stubbed LLM (no OpenRouter spend). - **Secret scan (gitleaks)** — fails the build on any new, un-ignored secret. -- **Container image scan (Grype)** — fails the build on a fixable CRITICAL CVE in +- **Container image scan (Grype)** — fails the build on a fixable CRITICAL or HIGH CVE in the sandbox image built from `config/default/sandbox/Containerfile` (HIGH and below are reported, not blocking). Findings upload to GitHub Security → Code scanning. A separate weekly scheduled scan (non-blocking) catches new CVEs diff --git a/Makefile b/Makefile index baa66671..4a53dfaa 100644 --- a/Makefile +++ b/Makefile @@ -120,11 +120,28 @@ test-cover: go test -coverprofile=coverage.out -covermode=atomic -p 1 -tags fleet_host_executor ./... @go tool cover -func=coverage.out | tail -1 -lint: lint-go lint-migrations +lint: lint-go lint-python lint-migrations lint-go: golangci-lint run +# lint-python: ruff over the 13 Python files (sandbox FileOp helper, python +# bridge, skill scripts, MCP test servers). Rule selection lives in ruff.toml so +# this and the CI job cannot disagree. +# +# Skips LOUDLY when ruff is absent rather than failing: not every contributor has +# it, and CI enforces the gate regardless (ci.yml + dev-ci.yml `python` job). The +# message names the install command so a local skip is a choice, not a surprise — +# a check that quietly does nothing is the failure mode this repo keeps writing +# post-mortems about. +lint-python: + @if command -v ruff >/dev/null 2>&1; then \ + ruff check . && ruff format --check . ; \ + else \ + echo "ruff not installed — SKIPPING the Python lint (CI still enforces it)."; \ + echo " install: python3 -m pip install --user 'ruff==0.15.8'"; \ + fi + # lint-migrations: reject dangerous DDL in NEW/CHANGED migration files (#256). # Diff-scoped (vs the merge-base with origin/main), so the existing corpus is # untouched; a no-op when no migration files changed or no base ref resolves. diff --git a/cmd/fleet/testdata/authstatus_server.py b/cmd/fleet/testdata/authstatus_server.py index e3d8448a..1b0df7eb 100644 --- a/cmd/fleet/testdata/authstatus_server.py +++ b/cmd/fleet/testdata/authstatus_server.py @@ -3,6 +3,7 @@ Advertises one auth-status tool. Set AUTH_FAIL=1 in the env to make the tools/call result carry isError=true (a failed upstream credential check). """ + import json import os import sys @@ -41,12 +42,16 @@ def main(): elif method == "tools/call": if FAIL: resp["result"] = { - "content": [{"type": "text", "text": "401 Unauthorized: key revoked"}], + "content": [ + {"type": "text", "text": "401 Unauthorized: key revoked"} + ], "isError": True, } else: resp["result"] = { - "content": [{"type": "text", "text": "authenticated: seat 12345 ok"}], + "content": [ + {"type": "text", "text": "authenticated: seat 12345 ok"} + ], "isError": False, } else: diff --git a/cmd/fleet/tls.go b/cmd/fleet/tls.go index 2d217e5e..87447690 100644 --- a/cmd/fleet/tls.go +++ b/cmd/fleet/tls.go @@ -106,6 +106,7 @@ func startRedirectServer(addr string, m *autocert.Manager) { func redirectToHTTPS(w http.ResponseWriter, r *http.Request) { target := "https://" + stripPort(r.Host) + r.URL.RequestURI() //nolint:gosec // G710: standard HTTP→HTTPS upgrade to the SAME Host (scheme-only); not an open redirect to an arbitrary origin. + // nosemgrep: go.lang.security.injection.open-redirect.open-redirect -- same reasoning as the G710 waiver above: the target is built from r.Host with the scheme forced to https, so it can only ever point back at the host the client already asked for. Not attacker-chosen. http.Redirect(w, r, target, http.StatusMovedPermanently) } diff --git a/docs/CODEQL.md b/docs/CODEQL.md new file mode 100644 index 00000000..8c7cc8b1 --- /dev/null +++ b/docs/CODEQL.md @@ -0,0 +1,520 @@ +# CodeQL: advanced setup, and the Go analysis that had stopped working + +Design note for the change that replaced GitHub's zero-config CodeQL "default +setup" with an advanced-setup workflow, `.github/workflows/codeql.yml`. + +Companion reading: [`NODE-TOOLCHAIN-HANDOFF.md`](NODE-TOOLCHAIN-HANDOFF.md) — +this is the same failure family (a toolchain version that had a second, +unreconciled copy) and the same rule applies: one declaration point per version, +asserted rather than remembered. [`TESTING.md`](TESTING.md) describes the rest of +the CI ladder. + +## What was broken + +Default setup's Go analysis failed on every main-targeting PR from the Go 1.27 +bump (#1240, promoted in #1242) onward. From PR #1245, job 96968001636: + +``` +Setup go version spec 1.26 +Found in cache @ /opt/hostedtoolcache/go/1.26.6/x64 +... +Run github/codeql-action/autobuild@v4 + env: GOTOOLCHAIN: local +... +go: go.mod requires go >= 1.27.0 (running go 1.26.6; GOTOOLCHAIN=local) +Failed to run `go mod tidy -e` in . +make: *** [Makefile:72: compile] Error 1 +Error running go tooling: exit status 1 +Extraction failed for all discovered Go projects. +CodeQL job status was configuration error. +``` + +Default setup installed the Go its extractor was built with (1.26.6) and ran +under `GOTOOLCHAIN=local`, so against a `go.mod` requiring 1.27 it could neither +use the local toolchain nor download a newer one. It also invokes `make`, which +puts this repo's Makefile on the autobuild path. + +The consequence is the part worth stating plainly: **the repo had no CodeQL +coverage of its Go code for that entire stretch.** The failure was loud (red +checks) but not blocking — `ci-gate` is the only required check on main — so it +was annotated in promote commit messages and lived on. + +Default setup is zero-config: it exposes no Go version input and no env, so there +was nothing to fix in place. Hence advanced setup. + +## A correction to the diagnosis + +The working assumption going in was that the `GOTOOLCHAIN=local` pin came from +GitHub's *generated* default-setup workflow, and that a plain advanced workflow +would therefore not have it at all. + +**That is wrong, and it was checked rather than assumed.** `GOTOOLCHAIN: local` +is set by `github/codeql-action` itself. It appears in the environment of four +separate steps in our own workflow's Go job — `Set up Go`, `Initialize CodeQL`, +`Autobuild`, and `Perform CodeQL Analysis`: + +``` +$ grep -rn 'GOTOOLCHAIN' 'Analyze (go)'/ +./3_Set up Go.txt line 74 +./4_Initialize CodeQL.txt line 15 +./5_Autobuild.txt line 10 +./6_Perform CodeQL Analysis.txt line 18 +``` + +So the fix is **not** "the pin is gone". The pin is still there. The fix is that +`actions/setup-go` makes the *local* toolchain 1.27.0, which is what `go.mod` +asks for, so `GOTOOLCHAIN=local` is satisfied instead of contradicted. The +autobuilder notices and proceeds: + +``` +Autobuilder was built with go1.26.5, environment has go1.27.0 +``` + +This matters for the next person: the mechanism is "give `local` something good +enough", not "unset the pin". Neither `env: GOTOOLCHAIN: auto` nor +`build-mode: manual` — the two fallbacks held in reserve — was needed. + +## What shipped + +`.github/workflows/codeql.yml`, one `analyze` job over a four-entry matrix. + +| language | build mode | queries | +| --- | --- | --- | +| `go` | `autobuild` | `security-extended` | +| `python` | `none` | `security-extended` | +| `javascript-typescript` | `none` | `security-extended` | +| `actions` | `none` | `security-extended` | + +**Security queries only — at the `security-extended` tier.** The code-quality +suite was enabled, measured, and deliberately removed (see "Why code quality +was dropped" below); the *security* side was then widened from the default +suite to `security-extended` once the default measured clean, so the broader +set also started from a zero baseline. + +Adopting the extended suite was a measurement, and it produced exactly **one +finding across all four languages**: `actions/untrusted-checkout/medium` on the +`workflow_call` checkout in `build-sandbox-image.yml`, whose `ref:` is fed by +the `fleet_ref` input. The query is a **name heuristic** — it flags any +checkout whose ref traces to a field matching `.*(head|branch|ref).*` — +reproduced locally with the CodeQL 2.26.3 bundle to confirm the trigger before +touching anything. Two honest responses existed and the in-code dismissal was +not one of them: the `actions` language ships **no `AlertSuppression.ql`**, so +there is no comment-waiver path at all. The finding was fixed for real instead: +a `pin` step now refuses `refs/pull/*` / `pull/*` refs (a fork PR ref would +carry fork-controlled code into a workflow that *executes the checked-out build +script*), and the checkout consumes that step's neutrally-named output. The +same hardening went into `publish-sandbox-image.yml` — the **unflagged twin** +that is strictly more dangerous (it holds `packages: write`) but escaped the +query because its ref plumbing was named differently. A heuristic query's +silence is not evidence of safety; the flagged file just pointed at the class. + +The extended suite then verified **clean in CI on all four languages** — Dev CI +run 525 (`32580031374`), the same run that exercises the hardened `actions` +lane — so the fail-on-findings gate holds at the extended tier, not just the +default one. + +`build-mode: none` is [not supported for +Go](https://docs.github.com/en/code-security/reference/code-scanning/codeql/build-options-for-compiled-languages) +— only `autobuild` or `manual` — so Go's toolchain has to be correct rather than +skipped. + +**The Go interpreter is resolved from `go.mod`,** via `actions/setup-go` with +`go-version-file: go.mod`, never a literal. A hardcoded `1.27` here would be the +same bug class #1240 and #1241 already fixed twice for node. + +That rule is enforced, and the enforcement was **already directory-wide**: +`scripts/check_versions_test.go`'s `TestWorkflowsDeclareVersionsByFile` walks +every `.github/workflows/*.yml` and fails on any literal `go-version:` / +`node-version:`. So `codeql.yml` came under the assertion the moment it was +added — **no test change was needed.** Verified by breaking it on purpose rather +than by reading the regex: + +``` +$ sed -i "s|go-version-file: go.mod|go-version: '1.27.0'|" .github/workflows/codeql.yml +$ go test -count=1 -run TestWorkflowsDeclareVersionsByFile ./scripts +--- FAIL: TestWorkflowsDeclareVersionsByFile (0.00s) + check_versions_test.go:149: codeql.yml pins a literal version ("go-version: '1") + — use `node-version-file: web/.nvmrc` or `go-version-file: go.mod` so the + version has one declaration point +``` + +### Triggers + +```yaml +push: branches: [main] +pull_request: branches: [main, dev] +schedule: - cron: '0 10 * * 1' +``` + +`push` on `main` mirrors `ci.yml` and produces the alert set of record for the +default branch. `pull_request` on `main` matches what default setup covered. + +`dev` on `pull_request` is the **one place this covers more than default setup +did**, and the expansion is deliberate. Every change lands on `dev` first; `main` +only ever receives a promote merge. Scanning `main` alone means a finding +surfaces for the first time on a promote commit — the same complaint +`dev-ci.yml`'s own header already makes about compilation ("a branch whose job is +to integrate should not be where compilation is first attempted"). It is also +what made this change provable before merge: with main-only triggers, the first +real run of a workflow written to fix a silent-failure bug would have happened +*after* it merged. + +There is no `push` trigger on `dev`: a push to `dev` is the merge of a PR that +was just scanned, so it would re-analyze identical content. + +The weekly cron exists because a CodeQL verdict is a function of the query pack +as well as the commit — new queries ship continuously, and without a schedule the +only way this repo learns that a newly published query flags existing code is +that some unrelated PR turns red. That is the argument +`govulncheck-scheduled.yml` already makes about `vuln.go.dev`. Monday 10:00 UTC +is offset from the 07:00 canary, the 08:00 daily govulncheck and the Monday 09:00 +Grype scan, following the same don't-contend-for-runners note those files carry. + +`dev-ci.yml`'s header used to list CodeQL among the checks deferred to the +dev→main gate. That is no longer true, so it now says where CodeQL runs instead. + +## Two defects in the first cut, both green + +Both were found by reading the run log. Both runs of this workflow were **green** +while neither behaviour worked. This is the failure mode the change exists to +fix, so it is worth being concrete: a passing CodeQL job proves nothing about +what was analyzed. + +### 1. `analysis-kinds` is not available to us + +The first attempt passed `analysis-kinds: code-scanning,code-quality`, on the +reading that one init call could build one database and run both suites. The +action emitted two `##[error]` lines — **and exited 0**: + +``` +The `analysis-kinds` input is experimental and for GitHub-internal use only. +[...] An analysis kind other than `code-scanning` was specified in a custom +workflow. This is not supported and will become a fatal error in a future +version of the CodeQL Action. If your intention is to use quality queries +outside of Code Quality, use the `queries` input with `code-quality` instead. + +[...] Specifying multiple values as input is no longer supported. Continuing +with only `analysis-kinds: code-scanning`. +``` + +Confirmed in the artifacts rather than trusted from the warning: the Go job +loaded only `codeql/go-queries`, evaluated 72 distinct queries, and uploaded a +single `go.sarif`. The code-quality half of the coverage this change claims to +restore was not running at all. + +Fixed at the time by using `queries: code-quality`, exactly as the message +directs. That is the working form, and it is worth recording for anyone who tries +`analysis-kinds` again and sees a green check: the input is accepted, two errors +are logged, and only security runs. + +Code quality was then measured and dropped — see the next section. The lasting +point from this defect is the one about evidence: the run was **green** with the +requested analysis silently not happening. + +### 2. Go extraction had one hole, and it was the worst file in the tree + +The first run extracted 426 files. The tree has 427 non-test `.go` files. The +missing one, by diffing the extractor's own file list against the checkout: + +``` +$ comm -13 extracted.txt local_nontest.txt +internal/sandbox/host.go +``` + +`internal/sandbox/host.go` is the **unsandboxed host executor** — bash and python +run directly on the host — fenced behind `//go:build fleet_host_executor` and so +absent from the default build. It is a CODEOWNERS-protected path, and both +`ci.yml` and `dev-ci.yml` deliberately pass that tag to `go vet` and `go test` +("Same tag as ci.yml so host.go [...] is vetted too") precisely so it is not +left unchecked. Leaving it as the single gap in Go coverage is not a defensible +default. + +`GOFLAGS: -tags=fleet_host_executor` on the autobuild step fixes it, matching the +lanes that already vet it. + +**The honest limit:** the total is still 426, because `host.go` and +`host_disabled.go` carry mutually exclusive build tags (`fleet_host_executor` and +`!fleet_host_executor`), so exactly one is ever in a build. The tag trades which +one CodeQL sees. That is a good trade — `host.go` is 410 lines of real +unsandboxed-execution logic, `host_disabled.go` is a 26-line refusal stub — but +it is a trade, not the elimination of a gap. Analyzing both would need two +databases. + +Note also that the extractor still logs `Build flags: ''`. `GOFLAGS` reaches the +`go` tooling through the environment, not through the extractor's own flag +plumbing, so the log line that looks like it should confirm the fix does not. +`host.go` appearing in the extracted list is what confirms it. + +## Why code quality was dropped + +The quality suite shipped first, ran, and was then removed on the evidence it +produced. Worth writing down, because "more queries" reads as strictly better +until you look at what they found. + +**What it found: 32 findings, every one note-level, and zero security findings.** + +| language | findings | what they were | +| --- | --- | --- | +| `go` | 2 | `go/useless-assignment-to-field` | +| `python` | 28 | `py/empty-except` (12), `py/implicit-string-concatenation-in-list` (9), `py/comparison-of-identical-expressions` (3), unused local/global/import (4) | +| `javascript-typescript` | 2 | `js/trivial-conditional`, `js/useless-assignment-to-local` | +| `actions` | 0 | — | + +Three reasons that adds up to "wrong tool", not "clean codebase": + +1. **For Go and the web tier it duplicates gates that already block.** + `golangci-lint` runs `gosec`, `staticcheck`, `revive`, `unparam`, `gocritic` + and more, and it is inside `ci-gate`; oxlint covers the web tier. + `go/useless-assignment-to-field` is squarely inside that remit. Paying ~40s of + CodeQL for a second opinion on it buys nothing. + +2. **28 of 32 were Python — a real gap, but ruff is the right instrument.** + Python genuinely had no linter (see `ruff.toml`), so those findings were the + suite's only unique contribution. ruff finds the same class of thing in well + under a second, with autofix, and now blocks. A slow job with no autofix is + the wrong shape for lint, especially for an agent expected to fix and re-push. + +3. **Three of the 32 were false positives on correct code.** + `py/comparison-of-identical-expressions` flagged `value != value` three times + in `bento_pdf.py` — the idiomatic NaN test, which is true only for NaN. + Enabling the equivalent ruff rule (`PLR0124`) was rejected for the same + reason. + +So the security queries stay (they are what nothing else here can do — see the +Semgrep comparison in `.github/workflows/semgrep.yml`), and quality moves to the +linters that were already gating. + +**What this costs, stated plainly:** the four Go/JS quality findings above are no +longer reported by anything, because `golangci-lint` and oxlint did not +independently flag them. That is a real, small loss of coverage accepted in +exchange for not running a second slow analyzer over ground three other tools +already cover. + +## What was verified + +Measured on run 2 (`7731615`, run `32571297663`), by downloading the run's log +archive and reading the extractor's and evaluator's own output — not from the +check mark. + +**Go extraction really happened:** + +``` +Found 2 go.mod files in: go.mod, web/go.mod. +Done running go list deps: resolved 916 packages. +Done extracting .../internal/sandbox/host.go +Success: extraction succeeded for all 2 discovered project(s). +``` + +426 distinct `.go` files extracted, and the set differs from the tree's 427 +non-test files by exactly `host_disabled.go`, per the build-tag trade above. + +**Every language produced a database, ran queries, and uploaded results.** +Distinct queries evaluated. The middle column is the security suite alone, which +is what ships; the right column is what adding `queries: code-quality` did, kept +here because it is the measurement the drop decision rests on: + +| language | security only (ships) | with code-quality (dropped) | SARIF | +| --- | --- | --- | --- | +| `go` | 72 | 116 (+44) | `go.sarif` | +| `python` | 90 | 292 (+202) | `python.sarif` | +| `javascript-typescript` | 178 | 374 (+196) | `javascript.sarif` | +| `actions` | 36 | 36 (+0) | `actions.sarif` | + +`actions` is unchanged **by design** — default setup ran it in the security +analysis only, and that was matched rather than widened. The new Go query +directories are `RedundantCode` and `InconsistentCode`; JavaScript gains +`Quality`; Python gains `Classes`, `Exceptions`, `Functions`, `Imports`, +`Lexical`, `Resources`, `Statements`, `Testing` and `Variables`. All four jobs +logged `Successfully uploaded results`. + +**`web/` is in scope and contributes nothing, by design.** The autobuilder +discovers both `go.mod` files and extracts both projects. `web/` reports: + +``` +Running extractor command '.../go-extractor [./...]' from directory 'web'. +No packages found. +Done running go list deps: resolved 0 packages. +``` + +That is the correct outcome and matches `web/go.mod`'s own comment — it is a +no-package boundary module that exists to stop root `go ... ./...` traversing Go +source vendored inside `node_modules`. It was left in scope rather than excluded: +extraction of an empty module is free, and excluding it would need a config file +whose only job is to suppress something harmless. **Verified, not assumed** — +this is the specific claim the old autobuild failure ("Extraction failed for all +discovered Go projects") made it reasonable to worry about. + +**The local gate**, on this branch: `make build`, `make lint` (0 issues), +`make test` (exit 0), `make lint-migrations` (no changed migrations). `make lint` +needed `golangci-lint` v2.13.1 built with Go 1.27 — the installed 2.5.0 was built +with go1.25.1 and cannot lint the tree. + +## What was NOT verified, and what is deliberately out of scope + +- **No push-on-`main` or scheduled run has executed.** Both triggers are + unexercised until this merges and is promoted. They are ordinary trigger + syntax, and the `pull_request` path shares every step with them, but the cron + expression itself has not fired. It is a weekly cron, so its first real proof + is up to a week after promotion. +- **`_test.go` files are not analyzed.** 621 test files are outside the + database, because `autobuild` builds packages, not tests. Default setup did + not analyze them either, so this is not a regression — it is an unchanged + limit, stated because "CodeQL covers the Go code" would otherwise overclaim. + Bringing tests in would need `build-mode: manual`. +- **The lines-of-code metric value was not read.** `Summary/LinesOfCode.ql` + evaluates, but CodeQL does not print the number to the job log; it lands in a + `.bqrs`. File and package counts are what was actually observed, so they are + what is reported here. No line count is claimed. +- **Alert counts on `main` are not claimed.** The zero-security-findings result + above was measured on a PR run of this branch. PR-run file-coverage detail is + suppressed by CodeQL ("To speed up pull request analysis, file coverage + information is only enabled when analyzing the default branch and protected + branches"), so the default-branch alert set is not established until this + merges and a promote lands on `main`. +- **`build-mode: manual` was not built.** `autobuild` works, so the more + complex option was not needed. If `autobuild` regresses, manual mode plus the + repo's own `go build ./...` is the fallback — and it is also the route to + analyzing test files. +- **The three existing `upload-sarif` calls were not touched** (`ci.yml`'s Grype + step, `govulncheck-scheduled.yml`, `grype-scheduled.yml`). They upload their + own SARIF independently of CodeQL configuration; breaking them would silently + drop CVE findings from the Security tab. + +## Two different things can gate, and they are not the same lever + +This distinction is the one most worth internalizing, because a status check on +the CodeQL job does **not** gate on findings: + +| you want to block a merge when… | the mechanism | where it lives | +| --- | --- | --- | +| the analysis **failed or did not run** | a required status check on `CodeQL gate` | branch protection / ruleset | +| CodeQL **found alerts** at/above a severity | **code scanning merge protection** | ruleset → "Code scanning" rule | + +The second is the one people mean by "gate on CodeQL", and the first does not +give it to you. **A CodeQL job with a hundred open alerts still exits 0 and +reports green** — the job's success only says extraction and query evaluation +worked. That is exactly why the toolchain break was able to hide for weeks behind +a red-but-not-required check, and equally why a green check is not evidence of a +clean codebase. + +fleet is a **public** repository, so code scanning merge protection is available +at no cost (on private repos it requires GitHub Advanced Security). To turn it +on: Settings → Rules → the "Main" ruleset → add the **Code scanning** rule → +add tool **CodeQL** → set the alert thresholds. Two independent knobs there: +*Security alerts* (the CWE/security queries — the only ones this workflow runs) +and *Alerts* (everything else, which would be where a code-quality suite landed +if one were enabled; it is not). Since the security suite currently reports zero +findings on this tree, a **High or higher** security threshold can go on without +inheriting a backlog. + +## Merge gating today — unchanged, with the lever put within reach + +**A finding now turns the check red.** A `Fail on findings` step fails the job on +any finding, at a threshold of *any* — safe to set because the security suite +currently reports zero on this tree, so there is no backlog to grandfather. +Without that step the analyze step exits 0 whether it found nothing or a hundred +alerts, so a red check could only ever mean "the scanner broke" — which is +exactly how the toolchain break hid for weeks. + +**A red check now blocks the merge too**, and through the *existing* required +check rather than a new one: `codeql.yml` is a reusable workflow +(`on: workflow_call`) that `ci.yml` and `dev-ci.yml` call as a job, and that +calling job sits in `ci-gate`'s / `Dev gate`'s `needs`. A correction worth +keeping: an earlier revision claimed this half needed a repo-settings click, +reasoning from "`needs` cannot cross workflow files" — true, but a +`workflow_call` brings the jobs into the caller's file, which is the standard +mechanism and what ships. + +It *cannot* be folded into `ci-gate`: a job's `needs` cannot reach across +workflow files. So `codeql.yml` carries its own aggregate **`CodeQL gate`** job, +mirroring `ci.yml`'s `CI gate` and `dev-ci.yml`'s `Dev gate`. That job is the one +deliberate piece of forward work here, and it is worth being clear that it +changes nothing on its own: + +- It does **not** make CodeQL required. Requiring a check is a repo-settings + action, deliberately not expressible from a workflow file. +- What it buys is that **flipping the switch later is one check, not four.** + Naming `Analyze (go)`, `Analyze (python)`, `Analyze (javascript-typescript)` + and `Analyze (actions)` individually in branch protection would mean + re-pointing branch protection by hand every time the matrix gains or loses a + language — and the failure mode of getting that wrong is the dangerous + direction: a required check that never reports again blocks every PR, or a + removed one silently stops gating. One aggregate check has neither problem. + +No ruleset action is required for any of this: the gate wiring above is entirely +in the workflow files. (`CodeQL gate` still exists as the aggregate job — the +weekly scheduled run's single verdict — and could additionally be named in the +ruleset as belt-and-braces, but nothing depends on that.) + +**On sequencing:** Not out of +caution for its own sake — because of this specific incident. The analysis spent +weeks red for a toolchain reason unrelated to any diff, and a required check in +that state blocks *every* merge, including the promote PR that would carry the +fix. Requiring it also means a `dev`-PR CodeQL failure blocks `dev`, a heavier +posture than that lane's stated "does it compile, lint, and pass tests" job. The +sequence with the least chance of self-inflicted deadlock is: merge this, watch a +few promotions go green, then add `CodeQL gate` to the ruleset. + +What makes that sequencing *safer than it was*: with code quality dropped, the +security suite is all that runs, and it currently reports **zero findings** on +this tree (see "Why code quality was dropped"). So there is no pre-existing +backlog for a required gate to trip over — which is the usual reason turning one +on hurts. The remaining risk is the one this whole document is about: a toolchain +or extractor regression going red for reasons unrelated to any diff. + +No repo-settings or API change to code-scanning configuration was attempted as +part of this change. + +## Where findings appear — and why the job log now says + +A CodeQL run reports **nothing about what it found** to its own log. It writes +SARIF, uploads it, exits 0, and the only lines resembling a result are +`Exporting results to SARIF...` and `Successfully uploaded results` — which say a +file moved, not what was in it. Verified by grepping a full run's log archive for +any alert or result count: there is none. + +That makes a run's actual outcome invisible to anyone reading CI output, to +`gh run view`, and to any automation holding the log but not the code-scanning +API. So the analyze step now also writes SARIF locally (`output:`) and a +following step jq-summarizes it into both the job log and the step summary — the +same thing `govulncheck-scheduled.yml` already does with its SARIF: + +``` +### CodeQL findings — go +2 [error] go/clear-text-logging +1 [warning] go/incomplete-hostname-regexp +1 [note] go/redundant-assignment +-- +total findings: 4 +``` + +It is reporting only and never fails the job; blocking on findings is merge +protection's job, above. When no SARIF was written it says so explicitly rather +than printing "No findings." — reporting a clean result you did not observe is +the error this repo keeps having to write down. + +`security-events: write` plus the analyze step's upload is the code-scanning +ingestion path, so results also land in the repo's **Security → Code scanning**. +From the run log: + +``` +Adding fingerprints to SARIF file. See ... sarif-support-for-code-scanning ... +##[group]Uploading code scanning results +Uploading results +Successfully uploaded results +Analysis upload status is complete. +``` + +Two practical consequences worth stating, because they explain an empty-looking +Security tab rather than a broken one: + +- **The Security tab's alert list is the DEFAULT BRANCH's.** This workflow's only + `push` trigger is `main`, so that list refreshes when a promote merge lands on + `main` — not when a PR is scanned. +- **PR runs report on the PR**, not into the default-branch alert list, and + CodeQL additionally suppresses file-coverage detail there: *"To speed up pull + request analysis, file coverage information is only enabled when analyzing the + default branch and protected branches."* + +So after this merges to `dev`, expect findings on subsequent PRs; expect the +Security tab's `main` list to repopulate at the next dev→main promotion. diff --git a/docs/SANDBOX-IMAGE-FRESHNESS.md b/docs/SANDBOX-IMAGE-FRESHNESS.md index b44e8b35..5e6b5a71 100644 --- a/docs/SANDBOX-IMAGE-FRESHNESS.md +++ b/docs/SANDBOX-IMAGE-FRESHNESS.md @@ -14,7 +14,7 @@ serving sandbox images built 6–7 weeks earlier, on top of an equally old That matters because a container image is frozen at build time. An unchanged Containerfile does not stop the base layers and packages *inside* the built image from aging and accumulating **published, already-fixed CVEs**. CI's -Grype gate (fail on a fixable CRITICAL) scans a **fresh** build of the +Grype gate (fail on a fixable CRITICAL or HIGH) scans a **fresh** build of the Containerfile — only an on-box rebuild ever brings a deployed box up to what CI vouched for. diff --git a/docs/SCANNING.md b/docs/SCANNING.md new file mode 100644 index 00000000..93064ba8 --- /dev/null +++ b/docs/SCANNING.md @@ -0,0 +1,295 @@ +# The scanning stack: who checks what, and what actually gates + +Design note for the change that stopped treating "add a scanner" as strictly +better and gave each tool the job it is actually good at. Companion to +[`CODEQL.md`](CODEQL.md) (why default setup was replaced, and why CodeQL now runs +security queries only) and [`TESTING.md`](TESTING.md) (the rest of the ladder). + +## The stack + +| tool | scope | speed | gates? | where results appear | +| --- | --- | --- | --- | --- | +| `golangci-lint` (incl. `gosec`) | Go lint + Go SAST patterns | ~30s | **blocks** (`ci-gate`) | job log | +| `oxlint` + `tsc` | web tier lint + types | ~5s | **blocks** (`ci-gate`) | job log | +| **`ruff`** | **Python lint** | **~1s** | **blocks** (`ci-gate`) | job log | +| `govulncheck` | Go dependency CVEs (called symbols) | ~30s | **blocks** (`ci-gate`) | job log + Security tab | +| `grype` | sandbox image CVEs (fixable **CRITICAL + HIGH**) | ~1m | **blocks** (`ci-gate`) | job log + Security tab | +| `gitleaks` | secrets, every branch | ~10s | **blocks** (`ci-gate`) | job log | +| **`npm audit`** | npm dependency CVEs (web + rampart-service) | ~5s | **blocks** (`ci-gate`) | job log | +| CodeQL | **interprocedural taint / `security-extended`** | ~2m | **blocks** (`ci-gate`/`Dev gate` via workflow_call) | job log + Security tab | +| **Semgrep** | **Go/JS/Python SAST + Actions supply chain** | ~40s | **blocks** (`ci-gate`/`Dev gate` via workflow_call) | job log + artifact | + +Two things were added here (**ruff**, **Semgrep**) and one was narrowed +(**CodeQL**, to security queries only). + +## Why each tool is where it is + +**The design rule: one owner per job.** A second tool over ground an existing +blocking gate already covers does not add safety — it adds a queue of duplicate +findings, and a scanner whose output is mostly already-adjudicated noise trains +people to close the tab. Every placement below follows from that. + +### ruff owns Python lint (new, blocking) + +fleet ships 13 Python files — the sandbox FileOp helper, the python bridge, the +bento-slides and data-profiler skill scripts, MCP test servers, icon/doc +generators — and **nothing linted any of them.** Go had `golangci-lint`, the web +tier had `oxlint`, Python had neither. Its only coverage was whatever CodeQL's +code-quality suite happened to notice, at ~40s and with no autofix. + +That gap was real: CodeQL's quality suite found 28 Python issues, its single +largest contribution anywhere. ruff finds the same class in under a second, with +autofix, and now blocks. + +The rule selection is measured, and `ruff.toml` records the numbers. The +default rules (`E4,E7,E9,F`) found 3 real findings, fixed on day one. The +**`B`, `SIM` and `S` (bandit) families were then measured (21 findings), all 21 +fixed, and the families enabled**: both `zip()` sites got `strict=True` (each +provably equal-length), the unclosed `NamedTemporaryFile` moved into its +`with`, the deliberate best-effort `try/except-pass` sites became explicit +`contextlib.suppress` with the intent stated at each, and the one subprocess +launch carries a reasoned `# noqa: S603` (argv is `sys.executable` plus +internal literals — mutation-tested: stripping the noqa re-fires the rule). +The pure-style tiers stay off: they are ~330 findings of `%`-format and +line-length churn with no correctness content. + +Three real findings were fixed to make the gate clean on day one, so a new +violation is a regression rather than noise in a backlog: + +- `internal/mcp/testdata/dummy_server.py` — unused `import os`. +- `bento_doc.py` — **a byte-identical duplicate `has_guard` definition.** Two + copies, one call site; the second silently shadowed the first. Dead code, and + the only finding here that was arguably a latent bug. +- `bento_pdf.py` — a lambda assigned to a name (`E731`), rewritten as a `def`. + +`ruff format --check` is **also gated** (CI and `make lint`): the whole tree +was ruff-formatted in one dedicated commit (9 files, ~3.7k lines, validated +against the full Go suite — the bento/fileops golden tests exercise these +scripts), so the gate started clean and a failure means one new file. + +### CodeQL owns interprocedural taint (narrowed, fails on findings) + +CodeQL is the only tool in this stack that does cross-function dataflow, and that +is exactly the shape of fleet's headline invariants: *a credential must not reach +a log sink, the model context, or the sandbox.* `go/clear-text-logging` is +literally that query. Nothing else here can express it. + +So CodeQL keeps its security queries and gives up everything else — the quality +suite duplicated `golangci-lint`/`oxlint` for Go and JS, and ruff is a better fit +for Python. Full reasoning and measurements in [`CODEQL.md`](CODEQL.md). + +It runs the **`security-extended`** suite — the broader security set, adopted +after the default suite measured clean — and reports **zero findings** on this +tree (verified in CI across all four languages on Dev CI run 525), which is +what makes it safe to gate: a `Fail on findings` step now fails the job on any +finding, so a red `Analyze (…)` check means the *code* has a problem rather than +just "the scanner broke". That distinction is the whole reason the Go toolchain +break sat unnoticed for weeks. + +Getting the extended suite to zero was itself a fix, not a rubber stamp: its +one finding across all four languages was `actions/untrusted-checkout/medium` +on `build-sandbox-image.yml`'s `fleet_ref`-fed checkout. Rather than waive it +(the `actions` language has no `AlertSuppression.ql`, so there is no in-code +waiver anyway), the workflow now **refuses `refs/pull/*` refs** before checking +out — a fork-PR ref would put fork-controlled code into a workflow that runs +the checked-out build script — and the identical hardening went into +`publish-sandbox-image.yml`, the *unflagged* twin that holds `packages: write` +and only escaped the (name-heuristic) query because its plumbing was named +differently. Details in [`CODEQL.md`](CODEQL.md). + +### Semgrep owns fast multi-language SAST + Actions supply chain (new, blocking) + +Semgrep is the opposite trade from CodeQL: seconds instead of minutes, no +database build, findings straight to stdout, rules cheap to write. That makes it +the right fit for an agent-driven loop. + +All four packs run — `p/github-actions`, `p/golang`, `p/javascript`, `p/python` — +and the lane **blocks** (`--error`, no `continue-on-error`). Getting there meant +fixing every real finding and adjudicating every false one. + +**The 51 real findings: mutable action tags. All fixed.** + +`p/github-actions` found one issue class nothing else in this repo checks — +actions referenced by a **mutable tag** (`actions/checkout@v7`) instead of an +immutable commit SHA. If a tag moves, attacker-controlled code runs with this +repo's `GITHUB_TOKEN`. Every one of the **53** action references across all 12 +workflows is now pinned: + +```yaml +uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 +``` + +Each SHA is the commit the previously-used tag resolved to at pin time, so the +pin is behaviourally identical to the runs already verified green — a pin should +not smuggle in a version bump. The trailing version comment is also the form +Dependabot reads and updates, and `.github/dependabot.yml` already watches the +`github-actions` ecosystem, so these stay current without hand-editing. + +Two `uses:` lines are deliberately left on `@main`: both are inside **comments** +in `build-sandbox-image.yml` / `publish-sandbox-image.yml`, documenting how a +downstream bundle repo calls fleet's reusable workflows. `@main` is the right +guidance for a consumer tracking fleet, and Semgrep does not flag them (a YAML +comment is not a `uses:` key). + +**The 6 false positives: suppressed at the line, with reasons.** + +Worth reading, because three were **already formally triaged and suppressed for +`gosec`** — which runs inside `golangci-lint` and already blocks — and one is +actively wrong: + +| finding | why it is wrong | +| --- | --- | +| `open-redirect` — `cmd/fleet/tls.go` | Standard HTTP→HTTPS upgrade to the **same** host. Already `//nolint:gosec G710`. | +| `math-random-used` — `internal/runner/runner.go` | `math/rand/v2`, used once, for ±10% jitter on a retry interval. | +| `cookie-missing-secure` — `internal/sched/handlers/elcano.go` | A **deletion** cookie (`Value=""`, `MaxAge=-1`), no secret; `Secure` is conditional so logout works over plain-HTTP dev. Already `//nolint:gosec G124`. | +| `unsafe-deserialization-interface` — `internal/mcp/httptool.go` | `json.Unmarshal` into `interface{}` is **required** — the value feeds a jq program over arbitrary JSON. A concrete struct cannot express "whatever shape the response had". | +| `x-frame-options-misconfiguration` — `web/src/proxy.ts` | The header value is the literal string `"DENY"`. No user input reaches it. | +| `insecure-file-permissions` — `internal/sandbox/fileops.py` | Advises `0o644` — **world-readable** — for a sandbox directory. Following it would be a security **regression**; `0750` is the file-tool contract. | + +Each carries a line-level `nosemgrep: ` naming the specific rule and the +reason. Scoped to the rule, so a *different* rule firing on the same line still +reports. + +**Every suppression was mutation-tested.** Removing it makes the finding +reappear; with it, the finding is gone. That matters because "0 findings" has two +explanations — the waivers work, or the rules silently stopped matching — and +only one of them is safety. Checked across all three comment syntaxes (Go `//`, +Python `#`, TypeScript `//`), including the one waiver that had to become a +*trailing* comment because a standalone comment inside a Go import block breaks +`goimports`. + +### npm audit owns dependency CVEs for the two npm trees (new, blocking) + +`govulncheck` is Go-only and `grype` scans the sandbox *image*, so the web +tier's dependency tree — and `scripts/rampart-service`'s — had no CVE gate at +all. `npm audit --audit-level=low` now runs in the `web` job of both CI lanes, +lockfile-only (no install needed), before the expensive `npm ci`, and fails on +**any** severity. Like govulncheck, its verdict is a function of the clock as +well as the commit: a new advisory can redden an unchanged tree, and that is +the point. + +Turning it on surfaced a real backlog immediately: + +- `web/` was already clean — 0 vulnerabilities — thanks to the steady stream of + merged Dependabot PRs. +- `scripts/rampart-service` **had no `package-lock.json` at all**, which meant + no reproducible installs and nothing for an auditor to read. Generating one + exposed **5 high-severity vulnerabilities** the missing lockfile had been + hiding: `sharp <0.35.0` (libvips CVE-2026-33327/-33328/-35590/-35591) and, + one layer down, `adm-zip <0.6.0` (GHSA-xcpc-8h2w-3j85, crafted-ZIP 4 GB + allocation) via `onnxruntime-node`. + +No upstream release fixes either — the latest `@huggingface/transformers` +still pins `sharp ^0.34.5`, and npm's own suggested "fix" was a breaking +*downgrade* of transformers — so `package.json` carries two `overrides` +(`sharp ^0.35.3`, `adm-zip ^0.6.0`, each the release immediately after the +vulnerable line). The overridden stack was **installed and load-tested**, not +just resolved: sharp renders a PNG through the new libvips, transformers loads +on it, rampart exports its API, and adm-zip 0.6 round-trips a zip. Audit result +after: 0 vulnerabilities in both trees. + +An override is a fork of upstream's intent, correct only while upstream is +broken — so `scripts/check-npm-overrides.sh` runs beside the audit in both +lanes and **fails the build the day upstream's own ranges reach the patched +lines**, with removal instructions. The reminder to drop the override is a red +build with a two-line fix, not stale-pin archaeology later. (Registry flake = +skip with a notice, never a verdict; mutation-tested in both directions. The +step invokes it as `"$GITHUB_WORKSPACE/scripts/check-npm-overrides.sh"` — the +job runs under `working-directory: web`, where a repo-relative path resolves +wrong; exit 127 on the first CI run taught that one.) + +## Findings are readable from the job log, on purpose + +Both scanners print a per-rule summary into the job log **and** the step summary: + +``` +### CodeQL findings — actions +[warning] actions/untrusted-checkout/medium .github/workflows/build-sandbox-image.yml:106 +-- +total findings: 1 +files in the actions database: 13 +``` + +Each line carries the **`file:line`** of the finding — an agent reading the log +can go straight to the site — and the `files in the … database` count is the +coverage line: "No findings." over an empty database is the green-but-vacuous +outcome this workflow exists to rule out, and the two are indistinguishable +without it. + +This exists because a CodeQL run reports **nothing** about what it found to its +own log — it writes SARIF, uploads it, exits 0, findings or not. Verified by +grepping a full run's log archive: there is no alert or result count anywhere. +That made a run's real outcome invisible to `gh run view` and to any agent +holding the log but not the code-scanning API. + +Semgrep additionally uploads its raw JSON as an artifact (`semgrep-findings`, +14-day retention), so a fixing agent can consume structured findings without +re-running the scan. The repo is public and these results are not sensitive; +withholding them buys nothing. + +The parse/scan-error line in that summary is at **zero**, and keeping it there +matters: a partial parse silently drops rules from a file. The three errors it +started with were all fixed for real — `${{ steps.build.outcome }}` interpolated +into a `run:` script in `build-sandbox-image.yml` (moved to `env:`, which is +also the injection-safe form), a `${tag:-(…)}` expansion default whose bare +paren choked the bash sub-parser (hoisted to a plain assignment), and an inline +`import("@playwright/test")` type in `fixtures.ts` (a named `import type`, +validated by `tsc`). + +## What gates — everything, through the gates that already exist + +Every lane in the table reaches the branch's aggregate gate: + +- `ci-gate` (the single required status check on `main`) `needs` the lint, test + and build jobs — **and the two scanners**. +- `Dev gate` does the same on `dev`. + +The scanners get there because `codeql.yml` and `semgrep.yml` are **reusable +workflows** (`on: workflow_call`): `ci.yml` and `dev-ci.yml` each call them as a +job, and a job that calls a reusable workflow sits in a gate's `needs` like any +other job. A scanner finding therefore blocks a merge through the existing +required check — **no branch-protection change, no new required check.** + +Worth recording as a correction: an earlier revision of this document claimed +gating the scanners required a repo-settings click, reasoning from "`needs` +cannot cross workflow files". True but incomplete — a `workflow_call` brings the +called jobs *into* the caller's file, which is the standard mechanism and what +ships now. The scanners' own `push`/`pull_request` triggers were removed so +nothing runs twice; each keeps its weekly `schedule` (new queries/rules against +unchanged code) and a `workflow_dispatch`. + +**Both scanners fail their job on any finding.** That is what makes a green +check mean "clean tree" rather than "the scanner ran" — the analyze step alone +exits 0 whether it found nothing or a hundred alerts, which is how the Go +toolchain break survived weeks behind a red-but-not-required check. Failing on +*any* finding is only defensible because the tree is at zero unsuppressed +findings everywhere — verified before the switch was flipped. A gate turned on +over an existing backlog is a gate people route around. + +(Code scanning merge protection — the ruleset's alert-severity rule — remains +available on top as a belt-and-braces option, but nothing depends on it now.) + +## Known gaps, deliberately not closed here + +Stated rather than left for rediscovery: + +- **`_test.go` files are outside CodeQL's database** (621 files) — `autobuild` + builds packages, not tests. Unchanged from default setup. +- **Semgrep's rule packs are registry-fetched and cannot be pinned by + vendoring** — investigated and rejected on license grounds, not neglect. The + Semgrep Rules License v1.0 grants use for "your own internal business + purposes" and states: *"This license does not allow you to distribute the + rules."* Committing them to this public MIT repo would be redistribution. + The binary version is pinned; the rules are not, so a registry-side rule + addition can turn CI red with no commit to blame — named here so a mystery + red Semgrep run has a first suspect. +- **A red scheduled scan now files an issue** (all four scheduled lanes) — a + cron failure has no PR to surface it, which is the rot pattern that let the + CodeQL toolchain break sit red for weeks. Deduped by title; re-failures + comment on the same issue. Mechanism differs by necessity: govulncheck and + grype carry an in-job step, while CodeQL and Semgrep are watched by + `scan-cron-alarm.yml` (a `workflow_run` watcher) — because a CALLED workflow + may not request permissions its caller did not grant, and the check fires at + plan time before any `if:` can skip the job. Learned by breaking it: an + `issues: write` alarm job inside the called workflows startup-failed the + entire calling Dev CI run. diff --git a/docs/TESTING.md b/docs/TESTING.md index 031877b9..dca9c52e 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -32,7 +32,7 @@ fix this doc (and the `make` targets) to match. | Go coverage | `go` | Coverage profile summarised in the log + job summary (advisory, no threshold) | `make test-cover` | | Go test -race | `go` | Race detector on the same suites | `make test-race` | | govulncheck | `go` | Dependency CVEs reachable from fleet | `make govulncheck` | -| Grype (image) | `grype-scan` | CVEs in the sandbox container image (fail on a fixable CRITICAL) | see below | +| Grype (image) | `grype-scan` | CVEs in the sandbox container image (fail on a fixable CRITICAL or HIGH) | see below | | Web lint/test/build | `web` | ESLint + vitest + `next build` | `make ci-web` | | Playwright (mocked) | `playwright` | Deterministic browser e2e, no backend | `make ci-e2e-mocked` | | Playwright (live) | `e2e-live` | Real stack + rootless-Podman sandbox, fake LLM | `npm run test:e2e:live` | diff --git a/docs/img/gen-open-source-vs-elcano.py b/docs/img/gen-open-source-vs-elcano.py index 6c8ad987..c4f7dc5a 100644 --- a/docs/img/gen-open-source-vs-elcano.py +++ b/docs/img/gen-open-source-vs-elcano.py @@ -6,20 +6,20 @@ OUT_DIR = os.path.dirname(os.path.abspath(__file__)) # ---------------------------------------------------------------- layout -PITCH = 142 # horizontal distance between item centers -C0 = 140 # x of first item center -N = 12 # total items -N_OSS = 6 # items under the open-source brace +PITCH = 142 # horizontal distance between item centers +C0 = 140 # x of first item center +N = 12 # total items +N_OSS = 6 # items under the open-source brace W = 2 * C0 + (N - 1) * PITCH H = 368 -TILE = 74 # tile side +TILE = 74 # tile side TILE_TOP = 116 TILE_CY = TILE_TOP + TILE / 2 LABEL_Y1 = TILE_TOP + TILE + 26 LABEL_Y2 = LABEL_Y1 + 20 -TOP_BRACE_Y = TILE_TOP - 12 # endpoints (just above tiles) +TOP_BRACE_Y = TILE_TOP - 12 # endpoints (just above tiles) TOP_BRACE_H = 12 TOP_LABEL_Y = TOP_BRACE_Y - TOP_BRACE_H * 2 - 16 @@ -28,16 +28,40 @@ BOT_LABEL_Y = BOT_BRACE_Y + BOT_BRACE_H * 2 + 34 THEMES = { - "light": dict(ink="#1f2328", text="#1f2328", muted="#59636e", - brace_top="#848d97", brace_bot="#59636e", - fill_op="0.16", stroke_op="0.5"), - "dark": dict(ink="#e6edf3", text="#e6edf3", muted="#9198a1", - brace_top="#767d86", brace_bot="#9198a1", - fill_op="0.22", stroke_op="0.6"), + "light": dict( + ink="#1f2328", + text="#1f2328", + muted="#59636e", + brace_top="#848d97", + brace_bot="#59636e", + fill_op="0.16", + stroke_op="0.5", + ), + "dark": dict( + ink="#e6edf3", + text="#e6edf3", + muted="#9198a1", + brace_top="#767d86", + brace_bot="#9198a1", + fill_op="0.22", + stroke_op="0.6", + ), } -ACCENTS = ["#3b82f6", "#10b981", "#8b5cf6", "#f59e0b", "#f43f5e", "#06b6d4", - "#8b5cf6", "#3b82f6", "#f59e0b", "#10b981", "#f43f5e", "#06b6d4"] +ACCENTS = [ + "#3b82f6", + "#10b981", + "#8b5cf6", + "#f59e0b", + "#f43f5e", + "#06b6d4", + "#8b5cf6", + "#3b82f6", + "#f59e0b", + "#10b981", + "#f43f5e", + "#06b6d4", +] FONT = "-apple-system, 'Segoe UI', 'Helvetica Neue', Arial, sans-serif" @@ -45,11 +69,13 @@ def sparkle(cx, cy, r): """Four-point star, filled with ink.""" k = r * 0.14 - return (f'') + return ( + f'' + ) # ---------------------------------------------------------------- glyphs @@ -62,17 +88,14 @@ def sparkle(cx, cy, r): '' '' '', - # 2 sandboxed tool calls (shield + prompt) '' '' '', - # 3 MCP connector catalog (plug) '' '' '', - # 4 any model (chip + sparkle) '' '' @@ -80,47 +103,37 @@ def sparkle(cx, cy, r): '' '' + sparkle(0, 0, 7), - # 5 budgets & audit (gauge) '' '' '', - # 6 web / TUI / API (terminal monitor) '' '' '' '', - # 7 custom MCP connectors (plug + sparkle) '' '' - '' - + sparkle(13, -12, 6), - + '' + sparkle(13, -12, 6), # 8 data integrations (database + arrow) '' '' '' '', - # 9 add-on capabilities (envelope + sparkle) '' - '' - + sparkle(14, -16, 6), - + '' + sparkle(14, -16, 6), # 10 forward-deployed engineering (person + map pin) '' '' '' '', - # 11 production-ready workflows (calendar + check) '' '' '' '', - # 12 support & operations (lifebuoy) '' '' @@ -148,13 +161,15 @@ def brace(x1, x2, y, h, up=True): """Curly brace from (x1,y) to (x2,y); cusp points up when up=True.""" s = -h if up else h xm = (x1 + x2) / 2 - return (f"M {x1} {y} " - f"C {x1} {y + s}, {x1 + h} {y + s}, {x1 + 2 * h} {y + s} " - f"L {xm - 2 * h} {y + s} " - f"C {xm - h} {y + s}, {xm} {y + s}, {xm} {y + 2 * s} " - f"C {xm} {y + s}, {xm + h} {y + s}, {xm + 2 * h} {y + s} " - f"L {x2 - 2 * h} {y + s} " - f"C {x2 - h} {y + s}, {x2} {y + s}, {x2} {y}") + return ( + f"M {x1} {y} " + f"C {x1} {y + s}, {x1 + h} {y + s}, {x1 + 2 * h} {y + s} " + f"L {xm - 2 * h} {y + s} " + f"C {xm - h} {y + s}, {xm} {y + s}, {xm} {y + 2 * s} " + f"C {xm} {y + s}, {xm + h} {y + s}, {xm + 2 * h} {y + s} " + f"L {x2 - 2 * h} {y + s} " + f"C {x2 - h} {y + s}, {x2} {y + s}, {x2} {y}" + ) def render(theme): @@ -172,12 +187,12 @@ def render(theme): glyph = GLYPHS[i].replace("{I}", t["ink"]) parts.append( f'' - f'' f'{glyph}' - f'' + f"" ) l1, l2 = (s.replace("&", "&") for s in LABELS[i]) parts.append( diff --git a/internal/clientconfig/builtin_skills/bento-slides/scripts/bento_doc.py b/internal/clientconfig/builtin_skills/bento-slides/scripts/bento_doc.py index b9c787fc..0f876038 100644 --- a/internal/clientconfig/builtin_skills/bento-slides/scripts/bento_doc.py +++ b/internal/clientconfig/builtin_skills/bento-slides/scripts/bento_doc.py @@ -28,6 +28,7 @@ """ import argparse +import contextlib import json import os import shutil @@ -306,10 +307,6 @@ def _inject_guard(raw): return raw[:at] + GUARD + raw[at:] -def has_guard(raw): - return GUARD_ID.encode() in raw - - def _decode_block(block): """Parse a document block's bytes into a dict. @@ -353,7 +350,6 @@ def _encode_block(doc): return encoded - # ── text fit (an estimate, because we have no font metrics) ────────────────── # # The app measures text for real and reports `text-overflow` from @@ -368,11 +364,17 @@ def _encode_block(doc): # a warning needs to clear the box by a margin before it prints. The app's own # validate() stays authoritative. _AVG_ADVANCE = 0.55 # mean glyph advance as a fraction of font size, sans-serif -_FIT_SLACK = 1.05 # only complain when the estimate clears the box by 5% +_FIT_SLACK = 1.05 # only complain when the estimate clears the box by 5% _ENTITIES = ( - ("—", "-"), ("–", "-"), (" ", " "), ("&", "&"), - ("<", "<"), (">", ">"), (""", '"'), ("'", "'"), + ("—", "-"), + ("–", "-"), + (" ", " "), + ("&", "&"), + ("<", "<"), + (">", ">"), + (""", '"'), + ("'", "'"), ) @@ -434,7 +436,8 @@ def _require(doc, key, kind, where): raise DeckError("%s: missing required field %r" % (where, key)) if not isinstance(doc[key], kind): raise DeckError( - "%s: field %r has the wrong type (%s)" % (where, key, type(doc[key]).__name__) + "%s: field %r has the wrong type (%s)" + % (where, key, type(doc[key]).__name__) ) return doc[key] @@ -539,21 +542,20 @@ def _write_atomic(path, data): leaves the original file exactly as it was. """ directory = os.path.dirname(os.path.abspath(path)) - fh = tempfile.NamedTemporaryFile( - dir=directory, prefix=".bento-", suffix=".tmp", delete=False - ) - tmp = fh.name + tmp = None try: - with fh: + with tempfile.NamedTemporaryFile( + dir=directory, prefix=".bento-", suffix=".tmp", delete=False + ) as fh: + tmp = fh.name fh.write(data) fh.flush() os.fsync(fh.fileno()) os.replace(tmp, path) except BaseException: - try: - os.unlink(tmp) - except OSError: - pass + if tmp is not None: + with contextlib.suppress(OSError): + os.unlink(tmp) raise @@ -664,7 +666,10 @@ def cmd_new(args): print("created %s — one title slide, ready to author" % path) print("offline-only deck: no update check, no live collaboration, no network") print("next: bento_doc.py get %s -o doc.json" % path) - print("download link (use this EXACT text, do not rebuild it): %s" % download_link(path)) + print( + "download link (use this EXACT text, do not rebuild it): %s" + % download_link(path) + ) return 0 @@ -772,12 +777,16 @@ def cmd_set(args): "- anyone holding an earlier copy can still join that room. The " "remedy for that is Share -> Rotate keys in the app.\n" % ( - " (including credential fields: %s)" % collab_field_label(dropped_fields) + " (including credential fields: %s)" + % collab_field_label(dropped_fields) if dropped_fields else "" ) ) - print("download link (use this EXACT text, do not rebuild it): %s" % download_link(args.deck)) + print( + "download link (use this EXACT text, do not rebuild it): %s" + % download_link(args.deck) + ) return 0 @@ -790,7 +799,9 @@ def cmd_validate(args): try: doc = json.loads(raw.decode("utf-8")) except ValueError as exc: - raise DeckError("%s is neither a deck nor valid JSON: %s" % (args.path, exc)) from exc + raise DeckError( + "%s is neither a deck nor valid JSON: %s" % (args.path, exc) + ) from exc if not isinstance(doc, dict): raise DeckError("%s must contain a JSON object" % args.path) kind = "document" @@ -843,7 +854,8 @@ def cmd_validate(args): "it joins that session with no click. Re-write it with `set` to " "remove the block and make the deck offline-only." % ( - " including credential fields (%s)" % collab_field_label(credential_fields) + " including credential fields (%s)" + % collab_field_label(credential_fields) if credential_fields else "" ) @@ -940,9 +952,10 @@ def cmd_pdf(args): out, pages, len(data) / 1024.0, - "" if not skipped + "" + if not skipped else " (%d hidden/state slide(s) left out, as in the app's own " - "export)" % skipped, + "export)" % skipped, ) ) for warning in warnings: @@ -965,7 +978,9 @@ def main(argv=None): p_new = sub.add_parser("new", help="start a deck from the bundled Bento app") p_new.add_argument("deck", help="path to create, e.g. decks/Q4_Review.bento.html") - p_new.add_argument("--title", help="deck title (default: derived from the filename)") + p_new.add_argument( + "--title", help="deck title (default: derived from the filename)" + ) p_new.set_defaults(func=cmd_new) p_get = sub.add_parser("get", help="extract a deck's document JSON") @@ -978,13 +993,17 @@ def main(argv=None): p_set.add_argument("doc", help="the document JSON to splice in") p_set.set_defaults(func=cmd_set) - p_val = sub.add_parser("validate", help="check a deck or document for format errors") + p_val = sub.add_parser( + "validate", help="check a deck or document for format errors" + ) p_val.add_argument("path") p_val.set_defaults(func=cmd_validate) p_pdf = sub.add_parser("pdf", help="render a deck's slides to a PDF you can attach") p_pdf.add_argument("deck") - p_pdf.add_argument("-o", "--output", help="PDF path (default: the deck's name + .pdf)") + p_pdf.add_argument( + "-o", "--output", help="PDF path (default: the deck's name + .pdf)" + ) p_pdf.set_defaults(func=cmd_pdf) args = parser.parse_args(argv) diff --git a/internal/clientconfig/builtin_skills/bento-slides/scripts/bento_pdf.py b/internal/clientconfig/builtin_skills/bento-slides/scripts/bento_pdf.py index 2fd3b9df..8ec13647 100644 --- a/internal/clientconfig/builtin_skills/bento-slides/scripts/bento_pdf.py +++ b/internal/clientconfig/builtin_skills/bento-slides/scripts/bento_pdf.py @@ -74,19 +74,30 @@ class PdfError(Exception): # ── colors ─────────────────────────────────────────────────────────────────── _NAMED = { - "transparent": (0, 0, 0, 0.0), "none": (0, 0, 0, 0.0), - "black": (0, 0, 0, 1.0), "white": (1, 1, 1, 1.0), - "red": (1, 0, 0, 1.0), "green": (0, 0.502, 0, 1.0), - "blue": (0, 0, 1, 1.0), "gray": (0.502, 0.502, 0.502, 1.0), - "grey": (0.502, 0.502, 0.502, 1.0), "silver": (0.753, 0.753, 0.753, 1.0), - "navy": (0, 0, 0.502, 1.0), "teal": (0, 0.502, 0.502, 1.0), - "orange": (1, 0.647, 0, 1.0), "yellow": (1, 1, 0, 1.0), - "purple": (0.502, 0, 0.502, 1.0), "inherit": None, "currentcolor": None, + "transparent": (0, 0, 0, 0.0), + "none": (0, 0, 0, 0.0), + "black": (0, 0, 0, 1.0), + "white": (1, 1, 1, 1.0), + "red": (1, 0, 0, 1.0), + "green": (0, 0.502, 0, 1.0), + "blue": (0, 0, 1, 1.0), + "gray": (0.502, 0.502, 0.502, 1.0), + "grey": (0.502, 0.502, 0.502, 1.0), + "silver": (0.753, 0.753, 0.753, 1.0), + "navy": (0, 0, 0.502, 1.0), + "teal": (0, 0.502, 0.502, 1.0), + "orange": (1, 0.647, 0, 1.0), + "yellow": (1, 1, 0, 1.0), + "purple": (0.502, 0, 0.502, 1.0), + "inherit": None, + "currentcolor": None, } _RGB_FN = re.compile( r"^rgba?\(\s*([0-9.]+%?)[\s,]+([0-9.]+%?)[\s,]+([0-9.]+%?)" - r"(?:[\s,/]+([0-9.]+%?))?\s*\)$", re.I) + r"(?:[\s,/]+([0-9.]+%?))?\s*\)$", + re.I, +) def _chan(tok): @@ -119,7 +130,7 @@ def parse_color(value, default=(0, 0, 0, 1.0)): if len(h) in (3, 4): vals = [int(c * 2, 16) / 255.0 for c in h] elif len(h) in (6, 8): - vals = [int(h[i:i + 2], 16) / 255.0 for i in range(0, len(h), 2)] + vals = [int(h[i : i + 2], 16) / 255.0 for i in range(0, len(h), 2)] else: return default except ValueError: @@ -150,130 +161,123 @@ def is_visible(rgba): # which has to guess at a font it will never see. Zeros are WinAnsi's unused # slots and never reached: text is transliterated into this encoding first. _WINANSI_WIDTHS_SRC = { - "Helvetica": - "278 278 355 556 556 889 667 191 333 333 389 584 278 333 278 278 556 " - "556 556 556 556 556 556 556 556 556 278 278 584 584 584 556 1015 667 " - "667 722 722 667 611 778 722 278 500 667 556 833 722 778 667 778 722 " - "667 611 722 667 944 667 667 611 278 278 278 469 556 333 556 556 500 " - "556 556 278 556 556 222 222 500 222 833 556 556 556 556 333 500 278 " - "556 500 722 500 500 500 334 260 334 584 350 556 350 222 556 333 1000 " - "556 556 333 1000 667 333 1000 350 611 350 350 222 222 333 333 350 " - "556 1000 333 1000 500 333 944 350 500 667 278 333 556 556 556 556 260 " - "556 333 737 370 556 584 333 737 552 400 549 333 333 333 576 537 278 " - "333 333 365 556 834 834 834 611 667 667 667 667 667 667 1000 722 667 " - "667 667 667 278 278 278 278 722 722 778 778 778 778 778 584 778 722 " - "722 722 722 667 667 611 556 556 556 556 556 556 889 500 556 556 556 " - "556 278 278 278 278 556 556 556 556 556 556 556 549 611 556 556 556 " - "556 500 556 500", - "Helvetica-Bold": - "278 333 474 556 556 889 722 238 333 333 389 584 278 333 278 278 556 " - "556 556 556 556 556 556 556 556 556 333 333 584 584 584 611 975 722 " - "722 722 722 667 611 778 722 278 556 722 611 833 722 778 667 778 722 " - "667 611 722 667 944 667 667 611 333 278 333 584 556 333 556 611 556 " - "611 556 333 611 611 278 278 556 278 889 611 611 611 611 389 556 333 " - "611 556 778 556 556 500 389 280 389 584 350 556 350 278 556 500 1000 " - "556 556 333 1000 667 333 1000 350 611 350 350 278 278 500 500 350 556 " - "1000 333 1000 556 333 944 350 500 667 278 333 556 556 556 556 280 556 " - "333 737 370 556 584 333 737 552 400 549 333 333 333 576 556 278 333 " - "333 365 556 834 834 834 611 722 722 722 722 722 722 1000 722 667 667 " - "667 667 278 278 278 278 722 722 778 778 778 778 778 584 778 722 722 " - "722 722 667 667 611 556 556 556 556 556 556 889 556 556 556 556 556 " - "278 278 278 278 611 611 611 611 611 611 611 549 611 611 611 611 611 " - "556 611 556", - "Helvetica-Oblique": - "278 278 355 556 556 889 667 191 333 333 389 584 278 333 278 278 556 " - "556 556 556 556 556 556 556 556 556 278 278 584 584 584 556 1015 667 " - "667 722 722 667 611 778 722 278 500 667 556 833 722 778 667 778 722 " - "667 611 722 667 944 667 667 611 278 278 278 469 556 333 556 556 500 " - "556 556 278 556 556 222 222 500 222 833 556 556 556 556 333 500 278 " - "556 500 722 500 500 500 334 260 334 584 350 556 350 222 556 333 1000 " - "556 556 333 1000 667 333 1000 350 611 350 350 222 222 333 333 350 " - "556 1000 333 1000 500 333 944 350 500 667 278 333 556 556 556 556 260 " - "556 333 737 370 556 584 333 737 552 400 549 333 333 333 576 537 278 " - "333 333 365 556 834 834 834 611 667 667 667 667 667 667 1000 722 667 " - "667 667 667 278 278 278 278 722 722 778 778 778 778 778 584 778 722 " - "722 722 722 667 667 611 556 556 556 556 556 556 889 500 556 556 556 " - "556 278 278 278 278 556 556 556 556 556 556 556 549 611 556 556 556 " - "556 500 556 500", - "Helvetica-BoldOblique": - "278 333 474 556 556 889 722 238 333 333 389 584 278 333 278 278 556 " - "556 556 556 556 556 556 556 556 556 333 333 584 584 584 611 975 722 " - "722 722 722 667 611 778 722 278 556 722 611 833 722 778 667 778 722 " - "667 611 722 667 944 667 667 611 333 278 333 584 556 333 556 611 556 " - "611 556 333 611 611 278 278 556 278 889 611 611 611 611 389 556 333 " - "611 556 778 556 556 500 389 280 389 584 350 556 350 278 556 500 1000 " - "556 556 333 1000 667 333 1000 350 611 350 350 278 278 500 500 350 556 " - "1000 333 1000 556 333 944 350 500 667 278 333 556 556 556 556 280 556 " - "333 737 370 556 584 333 737 552 400 549 333 333 333 576 556 278 333 " - "333 365 556 834 834 834 611 722 722 722 722 722 722 1000 722 667 667 " - "667 667 278 278 278 278 722 722 778 778 778 778 778 584 778 722 722 " - "722 722 667 667 611 556 556 556 556 556 556 889 556 556 556 556 556 " - "278 278 278 278 611 611 611 611 611 611 611 549 611 611 611 611 611 " - "556 611 556", - "Times-Roman": - "250 333 408 500 500 833 778 180 333 333 500 564 250 333 250 278 500 " - "500 500 500 500 500 500 500 500 500 278 278 564 564 564 444 921 722 " - "667 667 722 611 556 722 722 333 389 722 611 889 722 722 556 722 667 " - "556 611 722 722 944 722 722 611 333 278 333 469 500 333 444 500 444 " - "500 444 333 500 500 278 278 500 278 778 500 500 500 500 333 389 278 " - "500 500 722 500 500 444 480 200 480 541 350 500 350 333 500 444 1000 " - "500 500 333 1000 556 333 889 350 611 350 350 333 333 444 444 350 500 " - "1000 333 980 389 333 722 350 444 722 250 333 500 500 500 500 200 500 " - "333 760 276 500 564 333 760 500 400 549 300 300 333 576 453 250 333 " - "300 310 500 750 750 750 444 722 722 722 722 722 722 889 667 611 611 " - "611 611 333 333 333 333 722 722 722 722 722 722 722 564 722 722 722 " - "722 722 722 556 500 444 444 444 444 444 444 667 444 444 444 444 444 " - "278 278 278 278 500 500 500 500 500 500 500 549 500 500 500 500 500 " - "500 500 500", - "Times-Bold": - "250 333 555 500 500 1000 833 278 333 333 500 570 250 333 250 278 500 " - "500 500 500 500 500 500 500 500 500 333 333 570 570 570 500 930 722 " - "667 722 722 667 611 778 778 389 500 778 667 944 722 778 611 778 722 " - "556 667 722 722 1000 722 722 667 333 278 333 581 500 333 500 556 444 " - "556 444 333 500 556 278 333 556 278 833 556 500 556 556 444 389 333 " - "556 500 722 500 500 444 394 220 394 520 350 500 350 333 500 500 1000 " - "500 500 333 1000 556 333 1000 350 667 350 350 333 333 500 500 350 500 " - "1000 333 1000 389 333 722 350 444 722 250 333 500 500 500 500 220 500 " - "333 747 300 500 570 333 747 500 400 549 300 300 333 576 500 250 333 " - "300 330 500 750 750 750 500 722 722 722 722 722 722 1000 722 667 667 " - "667 667 389 389 389 389 722 722 778 778 778 778 778 570 778 722 722 " - "722 722 722 611 556 500 500 500 500 500 500 722 444 444 444 444 444 " - "278 278 278 278 500 556 500 500 500 500 500 549 500 556 556 556 556 " - "500 556 500", - "Times-Italic": - "250 333 420 500 500 833 778 214 333 333 500 675 250 333 250 278 500 " - "500 500 500 500 500 500 500 500 500 333 333 675 675 675 500 920 611 " - "611 667 722 611 611 722 722 333 444 667 556 833 667 722 611 722 611 " - "500 556 722 611 833 611 556 556 389 278 389 422 500 333 500 500 444 " - "500 444 278 500 500 278 278 444 278 722 500 500 500 500 389 389 278 " - "500 444 667 444 444 389 400 275 400 541 350 500 350 333 500 556 889 " - "500 500 333 1000 500 333 944 350 556 350 350 333 333 556 556 350 500 " - "889 333 980 389 333 667 350 389 556 250 389 500 500 500 500 275 500 " - "333 760 276 500 675 333 760 500 400 549 300 300 333 576 523 250 333 " - "300 310 500 750 750 750 500 611 611 611 611 611 611 889 667 611 611 " - "611 611 333 333 333 333 722 667 722 722 722 722 722 675 722 722 722 " - "722 722 556 611 500 500 500 500 500 500 500 667 444 444 444 444 444 " - "278 278 278 278 500 500 500 500 500 500 500 549 500 500 500 500 500 " - "444 500 444", - "Times-BoldItalic": - "250 389 555 500 500 833 778 278 333 333 500 570 250 333 250 278 500 " - "500 500 500 500 500 500 500 500 500 333 333 570 570 570 500 832 667 " - "667 667 722 667 667 722 778 389 500 667 611 889 722 722 611 722 667 " - "556 611 722 667 889 667 611 611 333 278 333 570 500 333 500 500 444 " - "500 444 333 444 500 278 278 444 278 722 500 500 500 500 389 389 278 " - "500 444 667 500 444 389 348 220 348 570 350 500 350 333 500 500 1000 " - "500 500 333 1000 556 333 944 350 611 350 350 333 333 500 500 350 500 " - "1000 333 1000 389 333 722 350 389 611 250 389 500 500 500 500 220 500 " - "333 747 266 500 606 333 747 500 400 549 300 300 333 576 500 250 333 " - "300 300 500 750 750 750 500 667 667 667 667 667 667 944 667 667 667 " - "667 667 389 389 389 389 722 722 722 722 722 722 722 570 722 722 722 " - "722 722 611 611 500 500 500 500 500 500 500 722 444 444 444 444 444 " - "278 278 278 278 500 500 500 500 500 500 500 549 500 500 500 500 500 " - "444 500 444", + "Helvetica": "278 278 355 556 556 889 667 191 333 333 389 584 278 333 278 278 556 " + "556 556 556 556 556 556 556 556 556 278 278 584 584 584 556 1015 667 " + "667 722 722 667 611 778 722 278 500 667 556 833 722 778 667 778 722 " + "667 611 722 667 944 667 667 611 278 278 278 469 556 333 556 556 500 " + "556 556 278 556 556 222 222 500 222 833 556 556 556 556 333 500 278 " + "556 500 722 500 500 500 334 260 334 584 350 556 350 222 556 333 1000 " + "556 556 333 1000 667 333 1000 350 611 350 350 222 222 333 333 350 " + "556 1000 333 1000 500 333 944 350 500 667 278 333 556 556 556 556 260 " + "556 333 737 370 556 584 333 737 552 400 549 333 333 333 576 537 278 " + "333 333 365 556 834 834 834 611 667 667 667 667 667 667 1000 722 667 " + "667 667 667 278 278 278 278 722 722 778 778 778 778 778 584 778 722 " + "722 722 722 667 667 611 556 556 556 556 556 556 889 500 556 556 556 " + "556 278 278 278 278 556 556 556 556 556 556 556 549 611 556 556 556 " + "556 500 556 500", + "Helvetica-Bold": "278 333 474 556 556 889 722 238 333 333 389 584 278 333 278 278 556 " + "556 556 556 556 556 556 556 556 556 333 333 584 584 584 611 975 722 " + "722 722 722 667 611 778 722 278 556 722 611 833 722 778 667 778 722 " + "667 611 722 667 944 667 667 611 333 278 333 584 556 333 556 611 556 " + "611 556 333 611 611 278 278 556 278 889 611 611 611 611 389 556 333 " + "611 556 778 556 556 500 389 280 389 584 350 556 350 278 556 500 1000 " + "556 556 333 1000 667 333 1000 350 611 350 350 278 278 500 500 350 556 " + "1000 333 1000 556 333 944 350 500 667 278 333 556 556 556 556 280 556 " + "333 737 370 556 584 333 737 552 400 549 333 333 333 576 556 278 333 " + "333 365 556 834 834 834 611 722 722 722 722 722 722 1000 722 667 667 " + "667 667 278 278 278 278 722 722 778 778 778 778 778 584 778 722 722 " + "722 722 667 667 611 556 556 556 556 556 556 889 556 556 556 556 556 " + "278 278 278 278 611 611 611 611 611 611 611 549 611 611 611 611 611 " + "556 611 556", + "Helvetica-Oblique": "278 278 355 556 556 889 667 191 333 333 389 584 278 333 278 278 556 " + "556 556 556 556 556 556 556 556 556 278 278 584 584 584 556 1015 667 " + "667 722 722 667 611 778 722 278 500 667 556 833 722 778 667 778 722 " + "667 611 722 667 944 667 667 611 278 278 278 469 556 333 556 556 500 " + "556 556 278 556 556 222 222 500 222 833 556 556 556 556 333 500 278 " + "556 500 722 500 500 500 334 260 334 584 350 556 350 222 556 333 1000 " + "556 556 333 1000 667 333 1000 350 611 350 350 222 222 333 333 350 " + "556 1000 333 1000 500 333 944 350 500 667 278 333 556 556 556 556 260 " + "556 333 737 370 556 584 333 737 552 400 549 333 333 333 576 537 278 " + "333 333 365 556 834 834 834 611 667 667 667 667 667 667 1000 722 667 " + "667 667 667 278 278 278 278 722 722 778 778 778 778 778 584 778 722 " + "722 722 722 667 667 611 556 556 556 556 556 556 889 500 556 556 556 " + "556 278 278 278 278 556 556 556 556 556 556 556 549 611 556 556 556 " + "556 500 556 500", + "Helvetica-BoldOblique": "278 333 474 556 556 889 722 238 333 333 389 584 278 333 278 278 556 " + "556 556 556 556 556 556 556 556 556 333 333 584 584 584 611 975 722 " + "722 722 722 667 611 778 722 278 556 722 611 833 722 778 667 778 722 " + "667 611 722 667 944 667 667 611 333 278 333 584 556 333 556 611 556 " + "611 556 333 611 611 278 278 556 278 889 611 611 611 611 389 556 333 " + "611 556 778 556 556 500 389 280 389 584 350 556 350 278 556 500 1000 " + "556 556 333 1000 667 333 1000 350 611 350 350 278 278 500 500 350 556 " + "1000 333 1000 556 333 944 350 500 667 278 333 556 556 556 556 280 556 " + "333 737 370 556 584 333 737 552 400 549 333 333 333 576 556 278 333 " + "333 365 556 834 834 834 611 722 722 722 722 722 722 1000 722 667 667 " + "667 667 278 278 278 278 722 722 778 778 778 778 778 584 778 722 722 " + "722 722 667 667 611 556 556 556 556 556 556 889 556 556 556 556 556 " + "278 278 278 278 611 611 611 611 611 611 611 549 611 611 611 611 611 " + "556 611 556", + "Times-Roman": "250 333 408 500 500 833 778 180 333 333 500 564 250 333 250 278 500 " + "500 500 500 500 500 500 500 500 500 278 278 564 564 564 444 921 722 " + "667 667 722 611 556 722 722 333 389 722 611 889 722 722 556 722 667 " + "556 611 722 722 944 722 722 611 333 278 333 469 500 333 444 500 444 " + "500 444 333 500 500 278 278 500 278 778 500 500 500 500 333 389 278 " + "500 500 722 500 500 444 480 200 480 541 350 500 350 333 500 444 1000 " + "500 500 333 1000 556 333 889 350 611 350 350 333 333 444 444 350 500 " + "1000 333 980 389 333 722 350 444 722 250 333 500 500 500 500 200 500 " + "333 760 276 500 564 333 760 500 400 549 300 300 333 576 453 250 333 " + "300 310 500 750 750 750 444 722 722 722 722 722 722 889 667 611 611 " + "611 611 333 333 333 333 722 722 722 722 722 722 722 564 722 722 722 " + "722 722 722 556 500 444 444 444 444 444 444 667 444 444 444 444 444 " + "278 278 278 278 500 500 500 500 500 500 500 549 500 500 500 500 500 " + "500 500 500", + "Times-Bold": "250 333 555 500 500 1000 833 278 333 333 500 570 250 333 250 278 500 " + "500 500 500 500 500 500 500 500 500 333 333 570 570 570 500 930 722 " + "667 722 722 667 611 778 778 389 500 778 667 944 722 778 611 778 722 " + "556 667 722 722 1000 722 722 667 333 278 333 581 500 333 500 556 444 " + "556 444 333 500 556 278 333 556 278 833 556 500 556 556 444 389 333 " + "556 500 722 500 500 444 394 220 394 520 350 500 350 333 500 500 1000 " + "500 500 333 1000 556 333 1000 350 667 350 350 333 333 500 500 350 500 " + "1000 333 1000 389 333 722 350 444 722 250 333 500 500 500 500 220 500 " + "333 747 300 500 570 333 747 500 400 549 300 300 333 576 500 250 333 " + "300 330 500 750 750 750 500 722 722 722 722 722 722 1000 722 667 667 " + "667 667 389 389 389 389 722 722 778 778 778 778 778 570 778 722 722 " + "722 722 722 611 556 500 500 500 500 500 500 722 444 444 444 444 444 " + "278 278 278 278 500 556 500 500 500 500 500 549 500 556 556 556 556 " + "500 556 500", + "Times-Italic": "250 333 420 500 500 833 778 214 333 333 500 675 250 333 250 278 500 " + "500 500 500 500 500 500 500 500 500 333 333 675 675 675 500 920 611 " + "611 667 722 611 611 722 722 333 444 667 556 833 667 722 611 722 611 " + "500 556 722 611 833 611 556 556 389 278 389 422 500 333 500 500 444 " + "500 444 278 500 500 278 278 444 278 722 500 500 500 500 389 389 278 " + "500 444 667 444 444 389 400 275 400 541 350 500 350 333 500 556 889 " + "500 500 333 1000 500 333 944 350 556 350 350 333 333 556 556 350 500 " + "889 333 980 389 333 667 350 389 556 250 389 500 500 500 500 275 500 " + "333 760 276 500 675 333 760 500 400 549 300 300 333 576 523 250 333 " + "300 310 500 750 750 750 500 611 611 611 611 611 611 889 667 611 611 " + "611 611 333 333 333 333 722 667 722 722 722 722 722 675 722 722 722 " + "722 722 556 611 500 500 500 500 500 500 500 667 444 444 444 444 444 " + "278 278 278 278 500 500 500 500 500 500 500 549 500 500 500 500 500 " + "444 500 444", + "Times-BoldItalic": "250 389 555 500 500 833 778 278 333 333 500 570 250 333 250 278 500 " + "500 500 500 500 500 500 500 500 500 333 333 570 570 570 500 832 667 " + "667 667 722 667 667 722 778 389 500 667 611 889 722 722 611 722 667 " + "556 611 722 667 889 667 611 611 333 278 333 570 500 333 500 500 444 " + "500 444 333 444 500 278 278 444 278 722 500 500 500 500 389 389 278 " + "500 444 667 500 444 389 348 220 348 570 350 500 350 333 500 500 1000 " + "500 500 333 1000 556 333 944 350 611 350 350 333 333 500 500 350 500 " + "1000 333 1000 389 333 722 350 389 611 250 389 500 500 500 500 220 500 " + "333 747 266 500 606 333 747 500 400 549 300 300 333 576 500 250 333 " + "300 300 500 750 750 750 500 667 667 667 667 667 667 944 667 667 667 " + "667 667 389 389 389 389 722 722 722 722 722 722 722 570 722 722 722 " + "722 722 611 611 500 500 500 500 500 500 500 722 444 444 444 444 444 " + "278 278 278 278 500 500 500 500 500 500 500 549 500 500 500 500 500 " + "444 500 444", } -WIDTHS = {name: [int(n) for n in src.split()] - for name, src in _WINANSI_WIDTHS_SRC.items()} +WIDTHS = { + name: [int(n) for n in src.split()] for name, src in _WINANSI_WIDTHS_SRC.items() +} WIDTHS["Courier"] = [600] * 224 WIDTHS["Courier-Bold"] = [600] * 224 WIDTHS["Courier-Oblique"] = [600] * 224 @@ -288,12 +292,40 @@ def is_visible(rgba): "Courier": (0.629, 0.157), } -_SERIF_HINTS = ("serif", "georgia", "times", "garamond", "fraunces", "playfair", - "merriweather", "cambria", "book", "charter", "spectral", "lora", - "source serif", "pt serif", "noto serif", "ibm plex serif", - "instrument serif", "newsreader", "literata", "bitter") -_MONO_HINTS = ("mono", "courier", "consolas", "menlo", "sf mono", "jetbrains", - "fira code", "source code", "ibm plex mono", "roboto mono") +_SERIF_HINTS = ( + "serif", + "georgia", + "times", + "garamond", + "fraunces", + "playfair", + "merriweather", + "cambria", + "book", + "charter", + "spectral", + "lora", + "source serif", + "pt serif", + "noto serif", + "ibm plex serif", + "instrument serif", + "newsreader", + "literata", + "bitter", +) +_MONO_HINTS = ( + "mono", + "courier", + "consolas", + "menlo", + "sf mono", + "jetbrains", + "fira code", + "source code", + "ibm plex mono", + "roboto mono", +) def family_of(stack): @@ -373,14 +405,34 @@ def is_bold(weight): # sign, plus-minus) is left alone — transliterating those would degrade text the # PDF can render perfectly. _TRANSLIT = { - "→": "->", "←": "<-", "↔": "<->", "⇒": "=>", "⇐": "<=", - "↑": "^", "↓": "v", "−": "-", "≤": "<=", "≥": ">=", - "≈": "~", "≠": "!=", "′": "'", "″": '"', - "✓": "*", "✔": "*", "✗": "x", "✘": "x", - "▶": ">", "◀": "<", "▪": "\u2022", "●": "\u2022", + "→": "->", + "←": "<-", + "↔": "<->", + "⇒": "=>", + "⇐": "<=", + "↑": "^", + "↓": "v", + "−": "-", + "≤": "<=", + "≥": ">=", + "≈": "~", + "≠": "!=", + "′": "'", + "″": '"', + "✓": "*", + "✔": "*", + "✗": "x", + "✘": "x", + "▶": ">", + "◀": "<", + "▪": "\u2022", + "●": "\u2022", # Exotic spaces a model can paste in: render as a normal space rather than # as a missing glyph in the middle of a headline. - "\u00a0": " ", "\u2007": " ", "\u2009": " ", "\u202f": " ", + "\u00a0": " ", + "\u2007": " ", + "\u2009": " ", + "\u202f": " ", } @@ -416,10 +468,11 @@ def warning(self): if not self.dropped: return None shown = " ".join("%r" % c for c in self.samples) - return ("%d character(s) have no glyph in the PDF core fonts and were " - "written as '?' (%s). Text outside Western European scripts " - "needs the deck's own Export PDF (print) button." - % (self.dropped, shown)) + return ( + "%d character(s) have no glyph in the PDF core fonts and were " + "written as '?' (%s). Text outside Western European scripts " + "needs the deck's own Export PDF (print) button." % (self.dropped, shown) + ) def text_width(encoder, text, face, size, letter_spacing=0.0): @@ -480,12 +533,19 @@ def push(chunk): nonlocal pending_break if not chunk: return - runs.append(Run(unescape(chunk), bold_depth > 0, italic_depth > 0, - mono_depth > 0, pending_break)) + runs.append( + Run( + unescape(chunk), + bold_depth > 0, + italic_depth > 0, + mono_depth > 0, + pending_break, + ) + ) pending_break = False for match in _TAG.finditer(text): - push(text[pos:match.start()]) + push(text[pos : match.start()]) pos = match.end() closing = match.group(1) == "/" name = match.group(2).lower() @@ -505,8 +565,7 @@ def push(chunk): mono_depth = max(0, mono_depth + step) push(text[pos:]) if pending_break: - runs.append(Run("", bold_depth > 0, italic_depth > 0, mono_depth > 0, - True)) + runs.append(Run("", bold_depth > 0, italic_depth > 0, mono_depth > 0, True)) return runs @@ -539,8 +598,9 @@ def add(self, piece): _SPLIT = re.compile(r"(\s+)") -def layout_text(encoder, runs, box_width, family, size, weight, - letter_spacing=0.0, wrap=True): +def layout_text( + encoder, runs, box_width, family, size, weight, letter_spacing=0.0, wrap=True +): """Greedy word wrap into `box_width`, honouring hard breaks and runs. Mirrors the browser closely enough to matter: `overflow-wrap: break-word` is @@ -569,8 +629,9 @@ def layout_text(encoder, runs, box_width, family, size, weight, continue if not wrap or blank or line.width + width <= box_width + 0.01: if wrap and blank and width > box_width + 0.01: - for part in _break_word(encoder, token, face, run_size, - box_width, letter_spacing): + for part in _break_word( + encoder, token, face, run_size, box_width, letter_spacing + ): if lines[-1].pieces: lines.append(Line()) lines[-1].add(part) @@ -581,8 +642,9 @@ def layout_text(encoder, runs, box_width, family, size, weight, line.width -= line.pieces.pop().width lines.append(Line()) if width > box_width + 0.01: - for part in _break_word(encoder, token, face, run_size, - box_width, letter_spacing): + for part in _break_word( + encoder, token, face, run_size, box_width, letter_spacing + ): if lines[-1].pieces: lines.append(Line()) lines[-1].add(part) @@ -640,8 +702,9 @@ def num(value): """Compact fixed-point number: PDF has no exponent notation.""" if value is None or not isinstance(value, (int, float)): return "0" - if isinstance(value, float) and (value != value or value in - (float("inf"), float("-inf"))): + if isinstance(value, float) and ( + value != value or value in (float("inf"), float("-inf")) + ): return "0" text = "%.4f" % value text = text.rstrip("0").rstrip(".") @@ -658,8 +721,9 @@ def reserve(self): return len(self._objects) - 1 def put(self, number, body): - self._objects[number] = body if isinstance(body, bytes) \ - else body.encode("latin-1") + self._objects[number] = ( + body if isinstance(body, bytes) else body.encode("latin-1") + ) def add(self, body): number = self.reserve() @@ -672,8 +736,7 @@ def add_stream(self, entries, data, compress=None): if compress: data = zlib.compress(data, 9) entries = entries + ["/Filter /FlateDecode"] - head = "<< %s /Length %d >>\nstream\n" % ( - " ".join(entries), len(data)) + head = "<< %s /Length %d >>\nstream\n" % (" ".join(entries), len(data)) return self.add(head.encode("latin-1") + data + b"\nendstream") def serialize(self, root, info): @@ -689,9 +752,10 @@ def serialize(self, root, info): out += b"0000000000 65535 f \n" for number in range(1, len(self._objects)): out += ("%010d 00000 n \n" % offsets[number]).encode("latin-1") - out += ("trailer\n<< /Size %d /Root %d 0 R /Info %d 0 R >>\n" - "startxref\n%d\n%%%%EOF\n" - % (len(self._objects), root, info, start)).encode("latin-1") + out += ( + "trailer\n<< /Size %d /Root %d 0 R /Info %d 0 R >>\n" + "startxref\n%d\n%%%%EOF\n" % (len(self._objects), root, info, start) + ).encode("latin-1") return bytes(out) @@ -709,12 +773,13 @@ def decode_data_uri(src): return None mime = (match.group(1) or "").lower() params = (match.group(2) or "").lower() - payload = src[match.end():] + payload = src[match.end() :] try: if "base64" in params: data = base64.b64decode(payload + "=" * (-len(payload) % 4)) else: from urllib.parse import unquote_to_bytes + data = unquote_to_bytes(payload) except (binascii.Error, ValueError): return None @@ -724,11 +789,20 @@ def decode_data_uri(src): class Image: """A decoded raster ready to become an XObject.""" - __slots__ = ("width", "height", "data", "filter", "colorspace", "bpc", - "smask", "palette") - - def __init__(self, width, height, data, filt, colorspace, bpc=8, - smask=None, palette=None): + __slots__ = ( + "width", + "height", + "data", + "filter", + "colorspace", + "bpc", + "smask", + "palette", + ) + + def __init__( + self, width, height, data, filt, colorspace, bpc=8, smask=None, palette=None + ): self.width = width self.height = height self.data = data @@ -750,9 +824,9 @@ def _jpeg_size(data): if marker in (0xD8, 0xD9) or 0xD0 <= marker <= 0xD7 or marker == 0x01: i += 2 continue - length = struct.unpack(">H", data[i + 2:i + 4])[0] + length = struct.unpack(">H", data[i + 2 : i + 4])[0] if 0xC0 <= marker <= 0xCF and marker not in (0xC4, 0xC8, 0xCC): - height, width = struct.unpack(">HH", data[i + 5:i + 9]) + height, width = struct.unpack(">HH", data[i + 5 : i + 9]) components = data[i + 9] return width, height, components i += 2 + length @@ -776,12 +850,13 @@ def _png_decode(data): palette = None trns = None while pos + 8 <= len(data): - length, kind = struct.unpack(">I4s", data[pos:pos + 8]) - body = data[pos + 8:pos + 8 + length] + length, kind = struct.unpack(">I4s", data[pos : pos + 8]) + body = data[pos + 8 : pos + 8 + length] pos += 12 + length if kind == b"IHDR": - width, height, depth, color, _comp, _filt, interlace = \ - struct.unpack(">IIBBBBB", body[:13]) + width, height, depth, color, _comp, _filt, interlace = struct.unpack( + ">IIBBBBB", body[:13] + ) header = (width, height, depth, color, interlace) elif kind == b"PLTE": palette = bytes(body) @@ -820,7 +895,7 @@ def _png_decode(data): for row in range(height): filt = raw[at] at += 1 - line = bytearray(raw[at:at + stride]) + line = bytearray(raw[at : at + stride]) at += stride if filt == 1: for i in range(unit, stride): @@ -838,14 +913,16 @@ def _png_decode(data): up = prev[i] upper_left = prev[i - unit] if i >= unit else 0 peak = left + up - upper_left - da, db, dc = (abs(peak - left), abs(peak - up), - abs(peak - upper_left)) - nearest = left if (da <= db and da <= dc) else \ - (up if db <= dc else upper_left) + da, db, dc = (abs(peak - left), abs(peak - up), abs(peak - upper_left)) + nearest = ( + left + if (da <= db and da <= dc) + else (up if db <= dc else upper_left) + ) line[i] = (line[i] + nearest) & 0xFF elif filt != 0: return "the PNG uses an unknown scanline filter (%d)" % filt - out[row * stride:(row + 1) * stride] = line + out[row * stride : (row + 1) * stride] = line prev = line if depth == 16: # keep the high byte; PDF viewers do the same visually @@ -863,7 +940,7 @@ def _png_decode(data): colour = bytearray() alpha = bytearray() for i in range(0, len(out), 4): - colour += out[i:i + 3] + colour += out[i : i + 3] alpha.append(out[i + 3]) space, ncomp = "/DeviceRGB", 3 elif color == 2: @@ -878,11 +955,20 @@ def _png_decode(data): smask = None if alpha is not None and min(alpha) < 255: - smask = Image(width, height, zlib.compress(bytes(alpha), 9), - "/FlateDecode", "/DeviceGray") + smask = Image( + width, height, zlib.compress(bytes(alpha), 9), "/FlateDecode", "/DeviceGray" + ) del ncomp - return Image(width, height, zlib.compress(bytes(colour), 9), - "/FlateDecode", space, 8, smask, palette) + return Image( + width, + height, + zlib.compress(bytes(colour), 9), + "/FlateDecode", + space, + 8, + smask, + palette, + ) def decode_image(src): @@ -902,8 +988,7 @@ def decode_image(src): if not size: return None, "the JPEG has no readable frame header" width, height, components = size - space = {1: "/DeviceGray", 3: "/DeviceRGB", 4: "/DeviceCMYK"}.get( - components) + space = {1: "/DeviceGray", 3: "/DeviceRGB", 4: "/DeviceCMYK"}.get(components) if space is None: return None, "the JPEG has %d components" % components return Image(width, height, data, "/DCTDecode", space), None @@ -942,7 +1027,8 @@ def font(self, face): if face not in self.fonts: number = self.writer.add( "<< /Type /Font /Subtype /Type1 /BaseFont /%s " - "/Encoding /WinAnsiEncoding >>" % face) + "/Encoding /WinAnsiEncoding >>" % face + ) self.fonts[face] = ("/F%d" % len(self.fonts), number) return self.fonts[face][0] @@ -950,39 +1036,46 @@ def alpha(self, fill_alpha, stroke_alpha): key = (round(fill_alpha, 3), round(stroke_alpha, 3)) if key not in self.gstates: number = self.writer.add( - "<< /Type /ExtGState /ca %s /CA %s >>" - % (num(key[0]), num(key[1]))) + "<< /Type /ExtGState /ca %s /CA %s >>" % (num(key[0]), num(key[1])) + ) self.gstates[key] = ("/GS%d" % len(self.gstates), number) return self.gstates[key][0] def image(self, image, cache_key=None): if cache_key is not None and cache_key in self._image_cache: return self._image_cache[cache_key] - entries = ["/Type /XObject", "/Subtype /Image", - "/Width %d" % image.width, "/Height %d" % image.height, - "/BitsPerComponent %d" % image.bpc, - "/Filter %s" % image.filter] + entries = [ + "/Type /XObject", + "/Subtype /Image", + "/Width %d" % image.width, + "/Height %d" % image.height, + "/BitsPerComponent %d" % image.bpc, + "/Filter %s" % image.filter, + ] if image.palette is not None: palette = self.writer.add_stream([], image.palette, compress=False) - entries.append("/ColorSpace [/Indexed /DeviceRGB %d %d 0 R]" - % (len(image.palette) // 3 - 1, palette)) + entries.append( + "/ColorSpace [/Indexed /DeviceRGB %d %d 0 R]" + % (len(image.palette) // 3 - 1, palette) + ) else: entries.append("/ColorSpace %s" % image.colorspace) if image.smask is not None: # The soft mask is referenced by the image, not by the page, so it # is written as a plain object and stays out of /XObject. mask = image.smask - head = ("<< /Type /XObject /Subtype /Image /Width %d /Height %d " - "/BitsPerComponent 8 /ColorSpace /DeviceGray /Filter %s " - "/Length %d >>\nstream\n" - % (mask.width, mask.height, mask.filter, len(mask.data))) + head = ( + "<< /Type /XObject /Subtype /Image /Width %d /Height %d " + "/BitsPerComponent 8 /ColorSpace /DeviceGray /Filter %s " + "/Length %d >>\nstream\n" + % (mask.width, mask.height, mask.filter, len(mask.data)) + ) number = self.writer.add( - head.encode("latin-1") + mask.data + b"\nendstream") + head.encode("latin-1") + mask.data + b"\nendstream" + ) entries.append("/SMask %d 0 R" % number) - head = "<< %s /Length %d >>\nstream\n" % ( - " ".join(entries), len(image.data)) - number = self.writer.add( - head.encode("latin-1") + image.data + b"\nendstream") + head = "<< %s /Length %d >>\nstream\n" % (" ".join(entries), len(image.data)) + number = self.writer.add(head.encode("latin-1") + image.data + b"\nendstream") name = "/Im%d" % len(self.xobjects) self.xobjects[name] = (name, number) if cache_key is not None: @@ -991,17 +1084,24 @@ def image(self, image, cache_key=None): def shading(self, coords, stops): """An axial (type 2) shading stitched from the gradient's stops.""" - stops = sorted(((max(0.0, min(1.0, at)), rgb) for at, rgb in stops), - key=lambda pair: pair[0]) + stops = sorted( + ((max(0.0, min(1.0, at)), rgb) for at, rgb in stops), + key=lambda pair: pair[0], + ) if len(stops) == 1: stops = [(0.0, stops[0][1]), (1.0, stops[0][1])] functions, bounds, encode = [], [], [] for index in range(len(stops) - 1): start, end = stops[index], stops[index + 1] - functions.append(self.writer.add( - "<< /FunctionType 2 /Domain [0 1] /C0 [%s] /C1 [%s] /N 1 >>" - % (" ".join(num(c) for c in start[1][:3]), - " ".join(num(c) for c in end[1][:3])))) + functions.append( + self.writer.add( + "<< /FunctionType 2 /Domain [0 1] /C0 [%s] /C1 [%s] /N 1 >>" + % ( + " ".join(num(c) for c in start[1][:3]), + " ".join(num(c) for c in end[1][:3]), + ) + ) + ) if index: bounds.append(stops[index][0]) encode.append("0 1") @@ -1011,12 +1111,17 @@ def shading(self, coords, stops): combined = self.writer.add( "<< /FunctionType 3 /Domain [0 1] /Functions [%s] " "/Bounds [%s] /Encode [%s] >>" - % (" ".join("%d 0 R" % f for f in functions), - " ".join(num(b) for b in bounds), " ".join(encode))) + % ( + " ".join("%d 0 R" % f for f in functions), + " ".join(num(b) for b in bounds), + " ".join(encode), + ) + ) number = self.writer.add( "<< /ShadingType 2 /ColorSpace /DeviceRGB /Coords [%s] " "/Function %d 0 R /Extend [true true] >>" - % (" ".join(num(c) for c in coords), combined)) + % (" ".join(num(c) for c in coords), combined) + ) name = "/Sh%d" % len(self.shadings) self.shadings[name] = (name, number) return name @@ -1036,24 +1141,31 @@ def gradient_mask(self, coords, stops, bbox): shading = self.shading(coords, greys) canvas_ops = "q %s sh Q" % shading form = self.writer.add_stream( - ["/Type /XObject", "/Subtype /Form", - "/BBox [%s]" % " ".join(num(v) for v in bbox), - "/Group << /Type /Group /S /Transparency /CS /DeviceGray >>", - "/Resources << /Shading << %s %d 0 R >> >>" - % (shading, self.shadings[shading][1])], - canvas_ops.encode("latin-1")) + [ + "/Type /XObject", + "/Subtype /Form", + "/BBox [%s]" % " ".join(num(v) for v in bbox), + "/Group << /Type /Group /S /Transparency /CS /DeviceGray >>", + "/Resources << /Shading << %s %d 0 R >> >>" + % (shading, self.shadings[shading][1]), + ], + canvas_ops.encode("latin-1"), + ) number = self.writer.add( "<< /Type /ExtGState /SMask << /S /Luminosity /G %d 0 R " - "/BC [0] >> >>" % form) + "/BC [0] >> >>" % form + ) name = "/GM%d" % len(self.gstates) self.gstates[name] = (name, number) return name def dictionary(self): def group(items): - return " ".join("%s %d 0 R" % (name, number) - for name, number in sorted(items, - key=lambda pair: pair[0])) + return " ".join( + "%s %d 0 R" % (name, number) + for name, number in sorted(items, key=lambda pair: pair[0]) + ) + parts = ["/ProcSet [/PDF /Text /ImageB /ImageC /ImageI]"] parts.append("/Font << %s >>" % group(self.fonts.values())) if self.xobjects: @@ -1130,8 +1242,7 @@ def rotate(self, degrees, cx, cy): radians = math.radians(degrees) cos, sin = math.cos(radians), math.sin(radians) self.translate(cx, cy) - self.op("%s %s %s %s 0 0 cm" - % (num(cos), num(sin), num(-sin), num(cos))) + self.op("%s %s %s %s 0 0 cm" % (num(cos), num(sin), num(-sin), num(cos))) self.translate(-cx, -cy) # -- paths ------------------------------------------------------------- @@ -1144,39 +1255,102 @@ def rect_path(self, x, y, w, h, radius=0): right, bottom = x + w, y + h self.op("%s %s m" % (num(x + radius), num(y))) self.op("%s %s l" % (num(right - radius), num(y))) - self.op("%s %s %s %s %s %s c" - % (num(right - radius + k), num(y), num(right), - num(y + radius - k), num(right), num(y + radius))) + self.op( + "%s %s %s %s %s %s c" + % ( + num(right - radius + k), + num(y), + num(right), + num(y + radius - k), + num(right), + num(y + radius), + ) + ) self.op("%s %s l" % (num(right), num(bottom - radius))) - self.op("%s %s %s %s %s %s c" - % (num(right), num(bottom - radius + k), - num(right - radius + k), num(bottom), - num(right - radius), num(bottom))) + self.op( + "%s %s %s %s %s %s c" + % ( + num(right), + num(bottom - radius + k), + num(right - radius + k), + num(bottom), + num(right - radius), + num(bottom), + ) + ) self.op("%s %s l" % (num(x + radius), num(bottom))) - self.op("%s %s %s %s %s %s c" - % (num(x + radius - k), num(bottom), num(x), - num(bottom - radius + k), num(x), num(bottom - radius))) + self.op( + "%s %s %s %s %s %s c" + % ( + num(x + radius - k), + num(bottom), + num(x), + num(bottom - radius + k), + num(x), + num(bottom - radius), + ) + ) self.op("%s %s l" % (num(x), num(y + radius))) - self.op("%s %s %s %s %s %s c" - % (num(x), num(y + radius - k), num(x + radius - k), num(y), - num(x + radius), num(y))) + self.op( + "%s %s %s %s %s %s c" + % ( + num(x), + num(y + radius - k), + num(x + radius - k), + num(y), + num(x + radius), + num(y), + ) + ) self.op("h") def ellipse_path(self, cx, cy, rx, ry): kx, ky = rx * _ARC_K, ry * _ARC_K self.op("%s %s m" % (num(cx + rx), num(cy))) - self.op("%s %s %s %s %s %s c" % (num(cx + rx), num(cy + ky), - num(cx + kx), num(cy + ry), - num(cx), num(cy + ry))) - self.op("%s %s %s %s %s %s c" % (num(cx - kx), num(cy + ry), - num(cx - rx), num(cy + ky), - num(cx - rx), num(cy))) - self.op("%s %s %s %s %s %s c" % (num(cx - rx), num(cy - ky), - num(cx - kx), num(cy - ry), - num(cx), num(cy - ry))) - self.op("%s %s %s %s %s %s c" % (num(cx + kx), num(cy - ry), - num(cx + rx), num(cy - ky), - num(cx + rx), num(cy))) + self.op( + "%s %s %s %s %s %s c" + % ( + num(cx + rx), + num(cy + ky), + num(cx + kx), + num(cy + ry), + num(cx), + num(cy + ry), + ) + ) + self.op( + "%s %s %s %s %s %s c" + % ( + num(cx - kx), + num(cy + ry), + num(cx - rx), + num(cy + ky), + num(cx - rx), + num(cy), + ) + ) + self.op( + "%s %s %s %s %s %s c" + % ( + num(cx - rx), + num(cy - ky), + num(cx - kx), + num(cy - ry), + num(cx), + num(cy - ry), + ) + ) + self.op( + "%s %s %s %s %s %s c" + % ( + num(cx + kx), + num(cy - ry), + num(cx + rx), + num(cy - ky), + num(cx + rx), + num(cy), + ) + ) self.op("h") def polygon_path(self, points): @@ -1187,11 +1361,14 @@ def polygon_path(self, points): def line_path(self, x1, y1, x2, y2): self.op("%s %s m %s %s l" % (num(x1), num(y1), num(x2), num(y2))) - def paint(self, fill=None, stroke=None, width=1.0, dash=None, cap=None, - even_odd=False): + def paint( + self, fill=None, stroke=None, width=1.0, dash=None, cap=None, even_odd=False + ): """Close out the current path with the right painting operator.""" - self.apply_alpha(fill[3] if is_visible(fill) else 1.0, - stroke[3] if is_visible(stroke) else 1.0) + self.apply_alpha( + fill[3] if is_visible(fill) else 1.0, + stroke[3] if is_visible(stroke) else 1.0, + ) if is_visible(fill): self.fill_color(fill) if is_visible(stroke) and width > 0: @@ -1268,7 +1445,9 @@ def content(self): _TOKEN = re.compile( r"\{\{\s*(page|pages|title|date|time|author|company|subject|event)" - r"(?::([^}]*))?\s*\}\}", re.I) + r"(?::([^}]*))?\s*\}\}", + re.I, +) def visible_slides(doc): @@ -1282,9 +1461,11 @@ def visible_slides(doc): slides = doc.get("slides") if not isinstance(slides, list): return [] - return [s for s in slides - if isinstance(s, dict) and not s.get("stateOf") - and not s.get("hidden")] + return [ + s + for s in slides + if isinstance(s, dict) and not s.get("stateOf") and not s.get("hidden") + ] def build_fields(doc, index, total, now): @@ -1332,8 +1513,9 @@ def one(match): def number(value, default=0.0): - return float(value) if isinstance(value, (int, float)) \ - and value == value else default + return ( + float(value) if isinstance(value, (int, float)) and value == value else default + ) class Renderer: @@ -1351,8 +1533,7 @@ def __init__(self, doc): self.theme_font = theme.get("fontFamily") or "sans-serif" self.theme_color = parse_color(theme.get("color"), (0.1, 0.1, 0.1, 1.0)) self.theme_bg = parse_color(theme.get("background"), (1, 1, 1, 1.0)) - self.assets = doc.get("assets") if isinstance(doc.get("assets"), - dict) else {} + self.assets = doc.get("assets") if isinstance(doc.get("assets"), dict) else {} self.writer = PdfWriter() self.res = Resources(self.writer) self.encoder = TextEncoder() @@ -1381,17 +1562,21 @@ def image_for(self, src): if not isinstance(resolved, str) or not resolved: return None if not resolved.startswith("data:"): - self.warn("image sources that are not embedded (%s...) were left " - "out: a PDF has no network, so only data: URIs and " - "doc.assets entries can be drawn." - % resolved[:32], "remote-image") + self.warn( + "image sources that are not embedded (%s...) were left " + "out: a PDF has no network, so only data: URIs and " + "doc.assets entries can be drawn." % resolved[:32], + "remote-image", + ) return None image, reason = decode_image(resolved) if image is None: - self.warn("an embedded image was skipped because %s. The built-in " - "export draws PNG (non-interlaced) and JPEG; re-embed it " - "in one of those, or use the deck's own PDF export." - % reason, "image-%s" % reason) + self.warn( + "an embedded image was skipped because %s. The built-in " + "export draws PNG (non-interlaced) and JPEG; re-embed it " + "in one of those, or use the deck's own PDF export." % reason, + "image-%s" % reason, + ) return None key = resolved if len(resolved) < 4096 else None name = self.res.image(image, cache_key=key) @@ -1401,8 +1586,10 @@ def image_for(self, src): def render(self): slides = visible_slides(self.doc) if not slides: - raise PdfError("the document has no printable slides (every slide " - "is hidden or a state variant)") + raise PdfError( + "the document has no printable slides (every slide " + "is hidden or a state variant)" + ) now = time.localtime() pages = [] contents = [] @@ -1414,22 +1601,31 @@ def render(self): pages.append(self.writer.reserve()) resources = self.res.dictionary() tree = self.writer.reserve() - for number_, content in zip(pages, contents): - self.writer.put(number_, - "<< /Type /Page /Parent %d 0 R /MediaBox " - "[0 0 %s %s] /Resources %d 0 R /Contents %d 0 R >>" - % (tree, num(self.page_width), - num(self.page_height), resources, content)) - self.writer.put(tree, - "<< /Type /Pages /Count %d /Kids [%s] >>" - % (len(pages), " ".join("%d 0 R" % p for p in pages))) + for number_, content in zip(pages, contents, strict=True): + self.writer.put( + number_, + "<< /Type /Page /Parent %d 0 R /MediaBox " + "[0 0 %s %s] /Resources %d 0 R /Contents %d 0 R >>" + % ( + tree, + num(self.page_width), + num(self.page_height), + resources, + content, + ), + ) + self.writer.put( + tree, + "<< /Type /Pages /Count %d /Kids [%s] >>" + % (len(pages), " ".join("%d 0 R" % p for p in pages)), + ) root = self.writer.add("<< /Type /Catalog /Pages %d 0 R >>" % tree) title = self.encoder.encode(str(self.doc.get("title") or "Bento deck")) info = self.writer.add( "<< /Title %s /Producer (fleet bento-slides skill) " "/CreationDate (D:%s) >>" - % (pdf_string(title).decode("latin-1"), - time.strftime("%Y%m%d%H%M%S", now))) + % (pdf_string(title).decode("latin-1"), time.strftime("%Y%m%d%H%M%S", now)) + ) note = self.encoder.warning() if note: self.warn(note, "encoding") @@ -1437,8 +1633,10 @@ def render(self): def render_slide(self, canvas, slide, fields): canvas.save() - canvas.op("%s 0 0 %s 0 %s cm" - % (num(self.scale), num(-self.scale), num(self.page_height))) + canvas.op( + "%s 0 0 %s 0 %s cm" + % (num(self.scale), num(-self.scale), num(self.page_height)) + ) background = parse_color(slide.get("background"), self.theme_bg) if is_visible(background): canvas.fill_color(background) @@ -1452,9 +1650,11 @@ def render_slide(self, canvas, slide, fields): except PdfError: raise except Exception as exc: # a bad element must not lose the deck - self.warn("element %r (%s) could not be drawn: %s" - % (element.get("id"), element.get("type"), exc), - "element-%s" % element.get("id")) + self.warn( + "element %r (%s) could not be drawn: %s" + % (element.get("id"), element.get("type"), exc), + "element-%s" % element.get("id"), + ) canvas.restore() def render_element(self, canvas, element, fields): @@ -1464,8 +1664,11 @@ def render_element(self, canvas, element, fields): w = number(element.get("w")) h = number(element.get("h")) opacity = element.get("opacity") - opacity = 1.0 if not isinstance(opacity, (int, float)) \ + opacity = ( + 1.0 + if not isinstance(opacity, (int, float)) else max(0.0, min(1.0, float(opacity))) + ) if opacity <= 0.001: return rotation = number(element.get("rotation")) @@ -1473,11 +1676,18 @@ def render_element(self, canvas, element, fields): if rotation: canvas.rotate(rotation, x + w / 2.0, y + h / 2.0) canvas.alpha(opacity) - if element.get("blur") or element.get("shadow") or \ - element.get("blend") or element.get("backdropFilter"): - self.warn("blur, drop shadow, blend and backdrop-filter effects " - "are not reproduced by the built-in export; the shapes " - "and text are drawn without them.", "filters") + if ( + element.get("blur") + or element.get("shadow") + or element.get("blend") + or element.get("backdropFilter") + ): + self.warn( + "blur, drop shadow, blend and backdrop-filter effects " + "are not reproduced by the built-in export; the shapes " + "and text are drawn without them.", + "filters", + ) box = (x, y, w, h) if kind == "text": self.render_text(canvas, element, box, fields) @@ -1492,12 +1702,14 @@ def render_element(self, canvas, element, fields): elif kind == "media": self.render_media(canvas, element, box) elif kind == "svg": - self.warn("an `svg` element was skipped: the built-in export has " - "no SVG renderer. Compose the artwork from shape " - "elements, or use the deck's own PDF export.", "svg") + self.warn( + "an `svg` element was skipped: the built-in export has " + "no SVG renderer. Compose the artwork from shape " + "elements, or use the deck's own PDF export.", + "svg", + ) else: - self.warn("unknown element type %r was skipped." % kind, - "type-%s" % kind) + self.warn("unknown element type %r was skipped." % kind, "type-%s" % kind) canvas.restore() # -- text -------------------------------------------------------------- @@ -1509,12 +1721,22 @@ def render_text(self, canvas, element, box, fields): return # print hides placeholders, and empty text draws nothing size = number(element.get("fontSize"), 24.0) or 24.0 line_height = element.get("lineHeight") - line_height = float(line_height) if isinstance( - line_height, (int, float)) and line_height > 0 else 1.2 + line_height = ( + float(line_height) + if isinstance(line_height, (int, float)) and line_height > 0 + else 1.2 + ) family = family_of(element.get("fontFamily") or self.theme_font) letter_spacing = number(element.get("letterSpacing")) - lines = layout_text(self.encoder, runs, w, family, size, - element.get("fontWeight"), letter_spacing) + lines = layout_text( + self.encoder, + runs, + w, + family, + size, + element.get("fontWeight"), + letter_spacing, + ) ascent, descent = VMETRICS[family] step = size * line_height block = step * len(lines) @@ -1534,8 +1756,11 @@ def render_text(self, canvas, element, box, fields): # background-clip:text. canvas.save() mode = 7 - elif isinstance(stroke, dict) and number(stroke.get("width")) > 0 \ - and stroke.get("fill") == "none": + elif ( + isinstance(stroke, dict) + and number(stroke.get("width")) > 0 + and stroke.get("fill") == "none" + ): color = parse_color(stroke.get("color"), color) if not is_visible(color) and not stops: return @@ -1549,16 +1774,26 @@ def render_text(self, canvas, element, box, fields): for piece in line.pieces: if not piece.text.strip(): continue - canvas.show_text(start + piece.x, baseline, piece.text, - piece.face, piece.size, letter_spacing) + canvas.show_text( + start + piece.x, + baseline, + piece.text, + piece.face, + piece.size, + letter_spacing, + ) canvas.end_text() if stops: # `ET` turns the accumulated glyph outlines (text render mode 7) # into the clip path, so the shading below paints THROUGH the # letters — the PDF equivalent of background-clip:text. - self.paint_gradient(canvas, gradient, stops, - self.gradient_coords(gradient, box), - (x, y, x + w, y + h)) + self.paint_gradient( + canvas, + gradient, + stops, + self.gradient_coords(gradient, box), + (x, y, x + w, y + h), + ) canvas.restore() def paint_gradient(self, canvas, gradient, stops, coords, bbox): @@ -1597,8 +1832,12 @@ def gradient_coords(self, gradient, box): radians = math.radians(angle) dx = math.sin(radians) / 2.0 dy = -math.cos(radians) / 2.0 - return (x + (0.5 - dx) * w, y + (0.5 - dy) * h, - x + (0.5 + dx) * w, y + (0.5 + dy) * h) + return ( + x + (0.5 - dx) * w, + y + (0.5 - dy) * h, + x + (0.5 + dx) * w, + y + (0.5 + dy) * h, + ) # ── SVG path data ──────────────────────────────────────────────────────────── @@ -1609,7 +1848,9 @@ def gradient_coords(self, gradient, box): # stroke width uniform, which is what the app's `vector-effect: # non-scaling-stroke` does in the browser. -_PATH_TOKEN = re.compile(r"([MmLlHhVvCcSsQqTtAaZz])|(-?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?)") +_PATH_TOKEN = re.compile( + r"([MmLlHhVvCcSsQqTtAaZz])|(-?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?)" +) def parse_path(data): @@ -1674,8 +1915,14 @@ def take(count): elif lower == "c": x1, y1, x2, y2, px, py = take(6) if relative: - x1, y1, x2, y2, px, py = (cx + x1, cy + y1, cx + x2, - cy + y2, cx + px, cy + py) + x1, y1, x2, y2, px, py = ( + cx + x1, + cy + y1, + cx + x2, + cy + y2, + cx + px, + cy + py, + ) out.append(("c", (x1, y1), (x2, y2), (px, py))) last_control = (x2, y2) cx, cy = px, py @@ -1702,20 +1949,22 @@ def take(count): if last_control is None: qx, qy = cx, cy else: - qx, qy = 2 * cx - last_control[0], \ - 2 * cy - last_control[1] - out.append(("c", - (cx + 2.0 / 3 * (qx - cx), cy + 2.0 / 3 * (qy - cy)), - (px + 2.0 / 3 * (qx - px), py + 2.0 / 3 * (qy - py)), - (px, py))) + qx, qy = 2 * cx - last_control[0], 2 * cy - last_control[1] + out.append( + ( + "c", + (cx + 2.0 / 3 * (qx - cx), cy + 2.0 / 3 * (qy - cy)), + (px + 2.0 / 3 * (qx - px), py + 2.0 / 3 * (qy - py)), + (px, py), + ) + ) last_control = (qx, qy) cx, cy = px, py elif lower == "a": rx, ry, rot, large, sweep, px, py = take(7) if relative: px, py = cx + px, cy + py - out.extend(_arc_to_beziers(cx, cy, rx, ry, rot, large, sweep, - px, py)) + out.extend(_arc_to_beziers(cx, cy, rx, ry, rot, large, sweep, px, py)) cx, cy = px, py else: break @@ -1741,8 +1990,7 @@ def _arc_to_beziers(x1, y1, rx, ry, rotation, large, sweep, x2, y2): scale = math.sqrt(lam) rx, ry = rx * scale, ry * scale denom = rx * rx * y1p * y1p + ry * ry * x1p * x1p - factor = 0.0 if denom == 0 else max( - 0.0, (rx * rx * ry * ry - denom) / denom) + factor = 0.0 if denom == 0 else max(0.0, (rx * rx * ry * ry - denom) / denom) coef = math.sqrt(factor) * (-1 if bool(large) == bool(sweep) else 1) cxp = coef * rx * y1p / ry cyp = -coef * ry * x1p / rx @@ -1769,8 +2017,10 @@ def angle_of(ux, uy): cos2, sin2 = math.cos(theta + step), math.sin(theta + step) def point(cos_t, sin_t): - return (cx + rx * cos_t * cos_phi - ry * sin_t * sin_phi, - cy + rx * cos_t * sin_phi + ry * sin_t * cos_phi) + return ( + cx + rx * cos_t * cos_phi - ry * sin_t * sin_phi, + cy + rx * cos_t * sin_phi + ry * sin_t * cos_phi, + ) px1, py1 = point(cos1, sin1) px2, py2 = point(cos2, sin2) @@ -1778,14 +2028,21 @@ def point(cos_t, sin_t): dy1 = -rx * sin1 * sin_phi + ry * cos1 * cos_phi dx2 = -rx * sin2 * cos_phi - ry * cos2 * sin_phi dy2 = -rx * sin2 * sin_phi + ry * cos2 * cos_phi - out.append(("c", (px1 + k * dx1, py1 + k * dy1), - (px2 - k * dx2, py2 - k * dy2), (px2, py2))) + out.append( + ( + "c", + (px1 + k * dx1, py1 + k * dy1), + (px2 - k * dx2, py2 - k * dy2), + (px2, py2), + ) + ) theta += step return out # ── shape, image, table and media elements ─────────────────────────────────── + def _dash_pattern(element, width): """The app's stroke-dasharray, in the same units.""" style = element.get("strokeStyle") @@ -1794,8 +2051,7 @@ def _dash_pattern(element, width): if style == "dotted": return [0.1, max(width * 2.2, 5)], 1 dash = element.get("strokeDash") - if style not in (None, "solid") and isinstance(dash, (int, float)) \ - and dash > 0: + if style not in (None, "solid") and isinstance(dash, (int, float)) and dash > 0: return [dash, dash], None return None, None @@ -1818,62 +2074,86 @@ def render_shape(self, canvas, element, box): canvas.save() canvas.translate(x, y) if shape == "rect": - canvas.rect_path(inset, inset, max(w - stroke_width, 0), - max(h - stroke_width, 0), - number(element.get("radius"))) + canvas.rect_path( + inset, + inset, + max(w - stroke_width, 0), + max(h - stroke_width, 0), + number(element.get("radius")), + ) elif shape == "ellipse": - canvas.ellipse_path(w / 2.0, h / 2.0, max(w / 2.0 - inset, 0), - max(h / 2.0 - inset, 0)) + canvas.ellipse_path( + w / 2.0, h / 2.0, max(w / 2.0 - inset, 0), max(h / 2.0 - inset, 0) + ) elif shape == "triangle": - canvas.polygon_path([(w / 2.0, inset), (w - inset, h - inset), - (inset, h - inset)]) + canvas.polygon_path( + [(w / 2.0, inset), (w - inset, h - inset), (inset, h - inset)] + ) elif shape == "arrow": shaft = h * 0.44 head = min(w * 0.38, h) top = (h - shaft) / 2.0 - canvas.polygon_path([(0, top), (w - head, top), (w - head, 0), - (w, h / 2.0), (w - head, h), - (w - head, top + shaft), (0, top + shaft)]) + canvas.polygon_path( + [ + (0, top), + (w - head, top), + (w - head, 0), + (w, h / 2.0), + (w - head, h), + (w - head, top + shaft), + (0, top + shaft), + ] + ) elif shape == "path": self.emit_path(canvas, element, w, h) else: - self.warn("unknown shape %r was skipped." % shape, - "shape-%s" % shape) + self.warn("unknown shape %r was skipped." % shape, "shape-%s" % shape) canvas.restore() return if stops: canvas.save() canvas.clip() - self.paint_gradient(canvas, element.get("fillGradient"), stops, - self.gradient_coords(element.get("fillGradient"), - (0, 0, w, h)), - (0, 0, w, h)) + self.paint_gradient( + canvas, + element.get("fillGradient"), + stops, + self.gradient_coords(element.get("fillGradient"), (0, 0, w, h)), + (0, 0, w, h), + ) canvas.restore() if is_visible(stroke) and stroke_width > 0: # The fill is painted by the shading, so re-lay the outline for # the stroke pass rather than trying to keep the clipped path. - self.render_shape_outline(canvas, element, box, stroke, - stroke_width, dash, cap) + self.render_shape_outline( + canvas, element, box, stroke, stroke_width, dash, cap + ) else: - canvas.paint(fill=fill, stroke=stroke, width=stroke_width, - dash=dash, cap=cap) + canvas.paint( + fill=fill, stroke=stroke, width=stroke_width, dash=dash, cap=cap + ) canvas.restore() - def render_shape_outline(self, canvas, element, box, stroke, width, dash, - cap): + def render_shape_outline(self, canvas, element, box, stroke, width, dash, cap): _, _, w, h = box inset = width / 2.0 shape = element.get("shape") or "rect" if shape == "rect": - canvas.rect_path(inset, inset, max(w - width, 0), max(h - width, 0), - number(element.get("radius"))) + canvas.rect_path( + inset, + inset, + max(w - width, 0), + max(h - width, 0), + number(element.get("radius")), + ) elif shape == "ellipse": - canvas.ellipse_path(w / 2.0, h / 2.0, max(w / 2.0 - inset, 0), - max(h / 2.0 - inset, 0)) + canvas.ellipse_path( + w / 2.0, h / 2.0, max(w / 2.0 - inset, 0), max(h / 2.0 - inset, 0) + ) elif shape == "triangle": - canvas.polygon_path([(w / 2.0, inset), (w - inset, h - inset), - (inset, h - inset)]) + canvas.polygon_path( + [(w / 2.0, inset), (w - inset, h - inset), (inset, h - inset)] + ) elif shape == "path": self.emit_path(canvas, element, w, h) else: @@ -1886,8 +2166,11 @@ def emit_path(self, canvas, element, w, h): if not segments: return pbox = element.get("pathBox") - if isinstance(pbox, list) and len(pbox) == 4 and \ - all(isinstance(v, (int, float)) for v in pbox): + if ( + isinstance(pbox, list) + and len(pbox) == 4 + and all(isinstance(v, (int, float)) for v in pbox) + ): vx, vy, vw, vh = (float(v) for v in pbox) else: vx, vy, vw, vh = 0.0, 0.0, w or 1.0, h or 1.0 @@ -1906,8 +2189,10 @@ def point(pair): canvas.op("%s %s l" % (num(px), num(py))) elif segment[0] == "c": (a, b), (c, d), (e, f) = (point(p) for p in segment[1:]) - canvas.op("%s %s %s %s %s %s c" - % (num(a), num(b), num(c), num(d), num(e), num(f))) + canvas.op( + "%s %s %s %s %s %s c" + % (num(a), num(b), num(c), num(d), num(e), num(f)) + ) elif segment[0] == "z": canvas.op("h") @@ -1918,21 +2203,36 @@ def render_line(self, canvas, element, box): color = parse_color(element.get("fill")) if not is_visible(color): return - tip = lambda kind: width * 2.6 if kind and kind != "none" else 0.0 + + def tip(kind): + return width * 2.6 if kind and kind != "none" else 0.0 + start = tip(element.get("lineStart")) end = tip(element.get("lineEnd")) mid = y + h / 2.0 dash, _ = _dash_pattern(element, width) canvas.save() canvas.line_path(x + start, mid, x + w - end, mid) - canvas.paint(stroke=color, width=width, dash=dash, - cap=0 if element.get("strokeStyle") == "dashed" else 1) - for kind, at_start in ((element.get("lineStart"), True), - (element.get("lineEnd"), False)): + canvas.paint( + stroke=color, + width=width, + dash=dash, + cap=0 if element.get("strokeStyle") == "dashed" else 1, + ) + for kind, at_start in ( + (element.get("lineStart"), True), + (element.get("lineEnd"), False), + ): if kind and kind != "none": - self.render_tip(canvas, kind, color, width, - x + (start if at_start else w - end), mid, - at_start) + self.render_tip( + canvas, + kind, + color, + width, + x + (start if at_start else w - end), + mid, + at_start, + ) canvas.restore() def render_tip(self, canvas, kind, color, width, at_x, at_y, reversed_): @@ -1943,15 +2243,19 @@ def render_tip(self, canvas, kind, color, width, at_x, at_y, reversed_): ref = 6.4 * unit tip_x = at_x + direction * (7.6 * unit - ref) back_x = at_x - direction * ref - canvas.polygon_path([ - (back_x, at_y - (4 - 0.4) * unit), - (tip_x, at_y), - (back_x, at_y + (7.6 - 4) * unit)]) + canvas.polygon_path( + [ + (back_x, at_y - (4 - 0.4) * unit), + (tip_x, at_y), + (back_x, at_y + (7.6 - 4) * unit), + ] + ) elif kind == "dot": canvas.ellipse_path(at_x, at_y, 2.6 * unit, 2.6 * unit) else: # bar - canvas.rect_path(at_x - 0.8 * unit, at_y - 3.6 * unit, - 1.6 * unit, 7.2 * unit) + canvas.rect_path( + at_x - 0.8 * unit, at_y - 3.6 * unit, 1.6 * unit, 7.2 * unit + ) canvas.paint(fill=color) def render_image(self, canvas, element, box): @@ -1967,11 +2271,15 @@ def render_image(self, canvas, element, box): canvas.rect_path(x, y, w, h, radius) canvas.clip() if fit in ("cover", "contain") and natural_w and natural_h: - scale = max(w / natural_w, h / natural_h) if fit == "cover" \ + scale = ( + max(w / natural_w, h / natural_h) + if fit == "cover" else min(w / natural_w, h / natural_h) + ) draw_w, draw_h = natural_w * scale, natural_h * scale - canvas.draw_image(name, x + (w - draw_w) / 2.0, - y + (h - draw_h) / 2.0, draw_w, draw_h) + canvas.draw_image( + name, x + (w - draw_w) / 2.0, y + (h - draw_h) / 2.0, draw_w, draw_h + ) else: canvas.draw_image(name, x, y, w, h) canvas.restore() @@ -1981,8 +2289,11 @@ def render_media(self, canvas, element, box): x, y, w, h = box kind = element.get("kind") radius = number(element.get("radius")) - backdrop = (0.043, 0.059, 0.078, 1.0) if kind == "video" \ + backdrop = ( + (0.043, 0.059, 0.078, 1.0) + if kind == "video" else (0.906, 0.929, 0.957, 1.0) + ) canvas.save() canvas.rect_path(x, y, w, h, radius) canvas.paint(fill=backdrop) @@ -1999,18 +2310,26 @@ def render_media(self, canvas, element, box): size = min(w, h) * 0.18 cx, cy = x + w / 2.0, y + h / 2.0 if kind == "video": - canvas.polygon_path([(cx - size * 0.4, cy - size * 0.55), - (cx + size * 0.6, cy), - (cx - size * 0.4, cy + size * 0.55)]) + canvas.polygon_path( + [ + (cx - size * 0.4, cy - size * 0.55), + (cx + size * 0.6, cy), + (cx - size * 0.4, cy + size * 0.55), + ] + ) canvas.paint(fill=glyph) else: - canvas.ellipse_path(cx - size * 0.25, cy + size * 0.35, - size * 0.28, size * 0.22) + canvas.ellipse_path( + cx - size * 0.25, cy + size * 0.35, size * 0.28, size * 0.22 + ) canvas.paint(fill=glyph) canvas.rect_path(cx, cy - size * 0.6, size * 0.12, size * 0.95) canvas.paint(fill=glyph) - self.warn("video and audio elements are drawn as a poster block — " - "a PDF cannot play media.", "media") + self.warn( + "video and audio elements are drawn as a poster block — " + "a PDF cannot play media.", + "media", + ) canvas.restore() @@ -2028,13 +2347,18 @@ def render_table(self, canvas, element, box): x, y, w, h = box columns = element.get("columns") rows = element.get("rows") - if not isinstance(columns, list) or not isinstance(rows, list) \ - or not columns or not rows: + if ( + not isinstance(columns, list) + or not isinstance(rows, list) + or not columns + or not rows + ): return - style = element.get("style") if isinstance(element.get("style"), - dict) else {} - weights = [max(0.0, number(c.get("w"))) if isinstance(c, dict) else 0.0 - for c in columns] + style = element.get("style") if isinstance(element.get("style"), dict) else {} + weights = [ + max(0.0, number(c.get("w"))) if isinstance(c, dict) else 0.0 + for c in columns + ] total_weight = sum(weights) or 1.0 widths = [w * weight / total_weight for weight in weights] header = bool(element.get("header")) @@ -2061,15 +2385,17 @@ def render_table(self, canvas, element, box): cell = cells[column_index] if column_index < len(cells) else {} cell = cell if isinstance(cell, dict) else {} bold = bool(cell.get("bold")) or is_header - inner = max(widths[column_index] - pad_x * 2 - border_width * 2, - 1.0) - lines = layout_text(self.encoder, - parse_inline(str(cell.get("html") or "")), - inner, family, font_size, - 700 if bold else 400) + inner = max(widths[column_index] - pad_x * 2 - border_width * 2, 1.0) + lines = layout_text( + self.encoder, + parse_inline(str(cell.get("html") or "")), + inner, + family, + font_size, + 700 if bold else 400, + ) laid.append((cell, lines, bold)) - tallest = max(tallest, len(lines) * font_size * line_height - + pad_y * 2) + tallest = max(tallest, len(lines) * font_size * line_height + pad_y * 2) grid.append(laid) natural.append(tallest) @@ -2092,17 +2418,19 @@ def render_table(self, canvas, element, box): if not is_header and style.get("zebra") and body_index % 2 == 1: zebra = style.get("zebra") left = x - for column_index, (cell, lines, bold) in enumerate(laid): + for column_index, (cell, lines, _bold) in enumerate(laid): width = widths[column_index] height = heights[row_index] background = cell.get("bg") or ( - style.get("headerBg") if is_header else zebra) + style.get("headerBg") if is_header else zebra + ) fill = parse_color(background, (0, 0, 0, 0.0)) if is_visible(fill) or (is_visible(border) and border_width): canvas.rect_path(left, top, width, height) canvas.paint(fill=fill, stroke=border, width=border_width) color = cell.get("color") or ( - style.get("headerColor") if is_header else style.get("color")) + style.get("headerColor") if is_header else style.get("color") + ) text_color = parse_color(color, self.theme_color) if not is_visible(text_color) or not lines: left += width @@ -2117,14 +2445,19 @@ def render_table(self, canvas, element, box): inner_width = max(width - pad_x * 2 - border_width * 2, 1.0) half_leading = (step - font_size * (ascent + descent)) / 2.0 for line_index, line in enumerate(lines): - baseline = (cell_top + line_index * step + half_leading - + ascent * font_size) + baseline = ( + cell_top + line_index * step + half_leading + ascent * font_size + ) start = inner_left + (inner_width - line.width) * align for piece in line.pieces: if piece.text.strip(): - canvas.show_text(start + piece.x, baseline, - piece.text, piece.face, - piece.size) + canvas.show_text( + start + piece.x, + baseline, + piece.text, + piece.face, + piece.size, + ) left += width top += heights[row_index] canvas.restore() @@ -2138,16 +2471,25 @@ def render_table(self, canvas, element, box): # placement. The engine deliberately honours only a subset of the ECharts option # shape, and so does this — a key the app ignores is a key we ignore. -CHART_COLORS = ["#5470c6", "#91cc75", "#fac858", "#ee6666", - "#73c0de", "#3ba272", "#fc8452", "#9a60b4"] +CHART_COLORS = [ + "#5470c6", + "#91cc75", + "#fac858", + "#ee6666", + "#73c0de", + "#3ba272", + "#fc8452", + "#9a60b4", +] AXIS_TEXT = "#6B7280" AXIS_LINE = (0.431, 0.471, 0.529, 0.45) SPLIT_LINE = (0.431, 0.471, 0.529, 0.15) def opt_num(value, default): - return float(value) if isinstance(value, (int, float)) \ - and value == value else default + return ( + float(value) if isinstance(value, (int, float)) and value == value else default + ) def format_number(value): @@ -2159,8 +2501,9 @@ def format_number(value): digits = text.lstrip("-") whole, _, fraction = digits.partition(".") grouped = "{:,}".format(int(whole)) - text = ("-" if negative else "") + grouped + ( - "." + fraction if fraction else "") + text = ( + ("-" if negative else "") + grouped + ("." + fraction if fraction else "") + ) return text @@ -2171,8 +2514,9 @@ def nice_ticks(low, high, count=5): span = high - low power = math.pow(10, math.floor(math.log10(span / count))) ratio = span / count / power - step = power * (10 if ratio >= 7.5 else 5 if ratio >= 3.5 - else 2 if ratio >= 1.5 else 1) + step = power * ( + 10 if ratio >= 7.5 else 5 if ratio >= 3.5 else 2 if ratio >= 1.5 else 1 + ) start = math.floor(low / step) * step stop = math.ceil(high / step) * step ticks = [] @@ -2193,8 +2537,9 @@ def magnitude(value): return 1 exponent = math.floor(math.log10(value)) mantissa = value / math.pow(10, exponent) - pick = 1 if mantissa <= 1 else 2 if mantissa <= 2 else \ - 5 if mantissa <= 5 else 10 + pick = ( + 1 if mantissa <= 1 else 2 if mantissa <= 2 else 5 if mantissa <= 5 else 10 + ) return pick * math.pow(10, exponent) slots = max(1, count) @@ -2217,14 +2562,20 @@ def axis_scale(values, axis, tick_count=None): top = axis.get("max") if axis.get("max") is not None else (high or 1.0) pinned = axis.get("min") is not None or axis.get("max") is not None if tick_count: - ticks = _even_ticks(bottom, top, tick_count) if pinned \ + ticks = ( + _even_ticks(bottom, top, tick_count) + if pinned else _step_ticks(bottom, top, tick_count - 1) + ) else: - ticks = _even_ticks(bottom, top, 6) if pinned \ - else nice_ticks(bottom, top) + ticks = _even_ticks(bottom, top, 6) if pinned else nice_ticks(bottom, top) formatter = axis.get("formatter") - labels = [formatter.replace("{value}", format_number(t)) if formatter - else format_number(t) for t in ticks] + labels = [ + formatter.replace("{value}", format_number(t)) + if formatter + else format_number(t) + for t in ticks + ] return {"lo": ticks[0], "hi": ticks[-1], "labels": labels} @@ -2234,16 +2585,20 @@ def normalize_chart(option, width, height): series = option.get("series") if isinstance(series, dict): series = [series] - series = [s for s in series if isinstance(s, dict)] \ - if isinstance(series, list) else [] + series = ( + [s for s in series if isinstance(s, dict)] if isinstance(series, list) else [] + ) is_pie = any(s.get("type") == "pie" for s in series) grid = option.get("grid") if isinstance(option.get("grid"), dict) else {} legend_option = option.get("legend") legend = None if legend_option: legend_option = legend_option if isinstance(legend_option, dict) else {} - text_style = legend_option.get("textStyle") \ - if isinstance(legend_option.get("textStyle"), dict) else {} + text_style = ( + legend_option.get("textStyle") + if isinstance(legend_option.get("textStyle"), dict) + else {} + ) legend = { "color": text_style.get("color") or AXIS_TEXT, "size": opt_num(text_style.get("fontSize"), 12), @@ -2252,13 +2607,15 @@ def normalize_chart(option, width, height): "itemHeight": opt_num(legend_option.get("itemHeight"), 10), "itemGap": opt_num(legend_option.get("itemGap"), 16), "top": legend_option.get("top") - if isinstance(legend_option.get("top"), (int, float)) else None, + if isinstance(legend_option.get("top"), (int, float)) + else None, "bottom": opt_num(legend_option.get("bottom"), 0), } band = max(legend["itemHeight"], legend["size"]) + 12 if legend else 0 x_axis = option.get("xAxis") if isinstance(option.get("xAxis"), dict) else {} - x_label = x_axis.get("axisLabel") \ - if isinstance(x_axis.get("axisLabel"), dict) else {} + x_label = ( + x_axis.get("axisLabel") if isinstance(x_axis.get("axisLabel"), dict) else {} + ) y_option = option.get("yAxis") if isinstance(y_option, dict): y_option = [y_option] @@ -2267,65 +2624,99 @@ def normalize_chart(option, width, height): y_axes = [] for axis in y_option[:2]: axis = axis if isinstance(axis, dict) else {} - label = axis.get("axisLabel") \ - if isinstance(axis.get("axisLabel"), dict) else {} - y_axes.append({ - "name": axis.get("name") if isinstance(axis.get("name"), str) - else None, - "min": axis.get("min") if isinstance(axis.get("min"), - (int, float)) else None, - "max": axis.get("max") if isinstance(axis.get("max"), - (int, float)) else None, - "formatter": label.get("formatter") - if isinstance(label.get("formatter"), str) else None, - "label": { - "color": label.get("color") or x_label.get("color") or AXIS_TEXT, - "size": opt_num(label.get("fontSize"), 12), - "weight": label.get("fontWeight", 400), - }, - }) + label = axis.get("axisLabel") if isinstance(axis.get("axisLabel"), dict) else {} + y_axes.append( + { + "name": axis.get("name") if isinstance(axis.get("name"), str) else None, + "min": axis.get("min") + if isinstance(axis.get("min"), (int, float)) + else None, + "max": axis.get("max") + if isinstance(axis.get("max"), (int, float)) + else None, + "formatter": label.get("formatter") + if isinstance(label.get("formatter"), str) + else None, + "label": { + "color": label.get("color") or x_label.get("color") or AXIS_TEXT, + "size": opt_num(label.get("fontSize"), 12), + "weight": label.get("fontWeight", 400), + }, + } + ) dual = not is_pie and len(y_axes) > 1 top_band = legend["top"] + band if legend and legend["top"] is not None else 0 - bottom_band = legend["bottom"] + band \ - if legend and legend["top"] is None else 0 + bottom_band = legend["bottom"] + band if legend and legend["top"] is None else 0 overflow = max(0.0, band - 12) - grid_bottom = opt_num(grid.get("bottom"), 44) if "bottom" in grid else \ - 44 + (overflow if legend and legend["top"] is None else 0) - grid_top = opt_num(grid.get("top"), 24) if "top" in grid else \ - 24 + (overflow if legend and legend["top"] is not None else 0) + grid_bottom = ( + opt_num(grid.get("bottom"), 44) + if "bottom" in grid + else 44 + (overflow if legend and legend["top"] is None else 0) + ) + grid_top = ( + opt_num(grid.get("top"), 24) + if "top" in grid + else 24 + (overflow if legend and legend["top"] is not None else 0) + ) if is_pie: - plot = {"x": 0.0, "y": top_band, "w": width, - "h": max(0.0, height - top_band - bottom_band)} + plot = { + "x": 0.0, + "y": top_band, + "w": width, + "h": max(0.0, height - top_band - bottom_band), + } else: left = opt_num(grid.get("left"), 48) right = opt_num(grid.get("right"), 56 if dual else 16) - plot = {"x": left, "y": grid_top, "w": width - left - right, - "h": max(0.0, height - grid_top - grid_bottom)} + plot = { + "x": left, + "y": grid_top, + "w": width - left - right, + "h": max(0.0, height - grid_top - grid_bottom), + } colors = option.get("color") - colors = [c for c in colors if isinstance(c, str)] \ - if isinstance(colors, list) and colors else CHART_COLORS - text_style = option.get("textStyle") \ - if isinstance(option.get("textStyle"), dict) else {} - axis_line = x_axis.get("axisLine") \ - if isinstance(x_axis.get("axisLine"), dict) else {} - axis_line_style = axis_line.get("lineStyle") \ - if isinstance(axis_line.get("lineStyle"), dict) else {} + colors = ( + [c for c in colors if isinstance(c, str)] + if isinstance(colors, list) and colors + else CHART_COLORS + ) + text_style = ( + option.get("textStyle") if isinstance(option.get("textStyle"), dict) else {} + ) + axis_line = ( + x_axis.get("axisLine") if isinstance(x_axis.get("axisLine"), dict) else {} + ) + axis_line_style = ( + axis_line.get("lineStyle") + if isinstance(axis_line.get("lineStyle"), dict) + else {} + ) first_y = y_option[0] if isinstance(y_option[0], dict) else {} - split = first_y.get("splitLine") \ - if isinstance(first_y.get("splitLine"), dict) else {} - split_style = split.get("lineStyle") \ - if isinstance(split.get("lineStyle"), dict) else {} + split = ( + first_y.get("splitLine") if isinstance(first_y.get("splitLine"), dict) else {} + ) + split_style = ( + split.get("lineStyle") if isinstance(split.get("lineStyle"), dict) else {} + ) pie = next((s for s in series if s.get("type") == "pie"), None) - pie_label = pie.get("label") if isinstance(pie, dict) and \ - isinstance(pie.get("label"), dict) else {} + pie_label = ( + pie.get("label") + if isinstance(pie, dict) and isinstance(pie.get("label"), dict) + else {} + ) categories = x_axis.get("data") return { - "w": width, "h": height, + "w": width, + "h": height, "font": family_of(text_style.get("fontFamily") or "sans-serif"), - "colors": colors, "series": series, "isPie": is_pie, + "colors": colors, + "series": series, + "isPie": is_pie, "categories": [str(c) for c in categories] - if isinstance(categories, list) else [], - "grid": plot, "legend": legend, + if isinstance(categories, list) + else [], + "grid": plot, + "legend": legend, "xAxisLabel": { "color": x_label.get("color") or y_axes[0]["label"]["color"], "size": opt_num(x_label.get("fontSize"), 12), @@ -2353,22 +2744,36 @@ def render_chart(self, canvas, element, box): if not chart["series"]: # The app draws the bare axis frame in this case, so we do too — # the warning is for the agent, not a reason to diverge. - self.warn("a chart element has no series; only its axes were " - "drawn (the app does the same).", "chart-empty") + self.warn( + "a chart element has no series; only its axes were " + "drawn (the app does the same).", + "chart-empty", + ) canvas.save() canvas.translate(x, y) if chart["isPie"]: self.draw_pie(canvas, chart) else: self.draw_cartesian(canvas, chart) - if chart["legend"] and any(s.get("name") or s.get("type") == "pie" - for s in chart["series"]): + if chart["legend"] and any( + s.get("name") or s.get("type") == "pie" for s in chart["series"] + ): self.draw_legend(canvas, chart) canvas.restore() # -- helpers ----------------------------------------------------------- - def chart_text(self, canvas, chart, cx, baseline, text, color, size, - anchor="middle", weight=400): + def chart_text( + self, + canvas, + chart, + cx, + baseline, + text, + color, + size, + anchor="middle", + weight=400, + ): """`si()` in the runtime: an SVG with a text-anchor.""" if text is None or text == "": return @@ -2379,17 +2784,22 @@ def chart_text(self, canvas, chart, cx, baseline, text, color, size, start = cx - width / 2.0 elif anchor == "end": start = cx - width - rgba = color if isinstance(color, tuple) \ + rgba = ( + color + if isinstance(color, tuple) else parse_color(color, (0.42, 0.45, 0.5, 1.0)) + ) canvas.apply_alpha(rgba[3]) canvas.fill_color(rgba) canvas.show_text(start, baseline, text, face, size) def series_color(self, chart, series, index): - item_style = series.get("itemStyle") \ - if isinstance(series.get("itemStyle"), dict) else {} - line_style = series.get("lineStyle") \ - if isinstance(series.get("lineStyle"), dict) else {} + item_style = ( + series.get("itemStyle") if isinstance(series.get("itemStyle"), dict) else {} + ) + line_style = ( + series.get("lineStyle") if isinstance(series.get("lineStyle"), dict) else {} + ) for candidate in (item_style.get("color"), line_style.get("color")): if isinstance(candidate, str): return parse_color(candidate) @@ -2406,28 +2816,34 @@ def draw_cartesian(self, canvas, chart): axis_count = len(chart["yAxes"]) def axis_index(series): - return min(axis_count - 1, - max(0, int(round(opt_num(series.get("yAxisIndex"), 0))))) + return min( + axis_count - 1, max(0, int(round(opt_num(series.get("yAxisIndex"), 0)))) + ) buckets = [[] for _ in range(axis_count)] for series in bars + lines: data = series.get("data") data = data if isinstance(data, list) else [] buckets[axis_index(series)].extend( - opt_num(value, 0) for value in data[:count]) + opt_num(value, 0) for value in data[:count] + ) low = float("inf") high = float("-inf") for series in points: for entry in series.get("data") or []: - px, py = (entry[0], entry[1]) if isinstance(entry, list) \ - and len(entry) >= 2 else (0, 0) + px, py = ( + (entry[0], entry[1]) + if isinstance(entry, list) and len(entry) >= 2 + else (0, 0) + ) low = min(low, px) high = max(high, px) buckets[0].append(opt_num(py, 0)) primary = axis_scale(buckets[0], chart["yAxes"][0]) ticks = len(primary["labels"]) - secondary = axis_scale(buckets[1], chart["yAxes"][1], ticks) \ - if axis_count > 1 else None + secondary = ( + axis_scale(buckets[1], chart["yAxes"][1], ticks) if axis_count > 1 else None + ) def value_y(value, axis=0): scale = secondary if axis == 1 and secondary else primary @@ -2441,40 +2857,66 @@ def zero_y(axis=0): for index in range(ticks): line_y = plot["y"] + plot["h"] - index / max(1, ticks - 1) * plot["h"] canvas.line_path(plot["x"], line_y, plot["x"] + plot["w"], line_y) - canvas.paint(stroke=chart["splitLine"]["color"], - width=chart["splitLine"]["width"]) + canvas.paint( + stroke=chart["splitLine"]["color"], width=chart["splitLine"]["width"] + ) label = chart["yAxes"][0]["label"] - self.chart_text(canvas, chart, plot["x"] - 8, - line_y + label["size"] * 0.35, - primary["labels"][index], - parse_color(label["color"], (0.42, 0.45, 0.5, 1)), - label["size"], "end", label["weight"]) + self.chart_text( + canvas, + chart, + plot["x"] - 8, + line_y + label["size"] * 0.35, + primary["labels"][index], + parse_color(label["color"], (0.42, 0.45, 0.5, 1)), + label["size"], + "end", + label["weight"], + ) if secondary: label2 = chart["yAxes"][1]["label"] - self.chart_text(canvas, chart, plot["x"] + plot["w"] + 8, - line_y + label2["size"] * 0.35, - secondary["labels"][index], - parse_color(label2["color"], - (0.42, 0.45, 0.5, 1)), - label2["size"], "start", label2["weight"]) - for axis, anchor, at_x in ((0, "end", plot["x"] - 8), - (1, "start", plot["x"] + plot["w"] + 8)): - if axis < axis_count and chart["yAxes"][axis].get("name") \ - and (axis == 0 or secondary): - self.chart_text(canvas, chart, at_x, plot["y"] - 9, - chart["yAxes"][axis]["name"], - parse_color(chart["xAxisLabel"]["color"], - (0.42, 0.45, 0.5, 1)), 11, anchor) + self.chart_text( + canvas, + chart, + plot["x"] + plot["w"] + 8, + line_y + label2["size"] * 0.35, + secondary["labels"][index], + parse_color(label2["color"], (0.42, 0.45, 0.5, 1)), + label2["size"], + "start", + label2["weight"], + ) + for axis, anchor, at_x in ( + (0, "end", plot["x"] - 8), + (1, "start", plot["x"] + plot["w"] + 8), + ): + if ( + axis < axis_count + and chart["yAxes"][axis].get("name") + and (axis == 0 or secondary) + ): + self.chart_text( + canvas, + chart, + at_x, + plot["y"] - 9, + chart["yAxes"][axis]["name"], + parse_color(chart["xAxisLabel"]["color"], (0.42, 0.45, 0.5, 1)), + 11, + anchor, + ) baseline = zero_y(0) canvas.line_path(plot["x"], baseline, plot["x"] + plot["w"], baseline) - canvas.paint(stroke=chart["axisLine"]["color"], - width=chart["axisLine"]["width"]) + canvas.paint( + stroke=chart["axisLine"]["color"], width=chart["axisLine"]["width"] + ) # A scatter with no categories gets a numeric x axis, like the app. if points and not categories: - span_ticks = nice_ticks(0 if low == float("inf") else min(0, low), - 1 if high == float("-inf") else high) + span_ticks = nice_ticks( + 0 if low == float("inf") else min(0, low), + 1 if high == float("-inf") else high, + ) first, last = span_ticks[0], span_ticks[-1] width = (last - first) or 1 @@ -2482,29 +2924,35 @@ def point_x(value): return plot["x"] + (value - first) / width * plot["w"] for tick in span_ticks: - self.chart_text(canvas, chart, point_x(tick), - plot["y"] + plot["h"] - + chart["xAxisLabel"]["size"] + 6, - format_number(tick), - parse_color(chart["xAxisLabel"]["color"], - (0.42, 0.45, 0.5, 1)), - chart["xAxisLabel"]["size"], "middle", - chart["xAxisLabel"]["weight"]) + self.chart_text( + canvas, + chart, + point_x(tick), + plot["y"] + plot["h"] + chart["xAxisLabel"]["size"] + 6, + format_number(tick), + parse_color(chart["xAxisLabel"]["color"], (0.42, 0.45, 0.5, 1)), + chart["xAxisLabel"]["size"], + "middle", + chart["xAxisLabel"]["weight"], + ) for index, series in enumerate(points): - color = self.series_color(chart, series, - chart["series"].index(series)) + color = self.series_color(chart, series, chart["series"].index(series)) radius = opt_num(series.get("symbolSize"), 10) / 2.0 for entry in series.get("data") or []: - px, py = (entry[0], entry[1]) \ - if isinstance(entry, list) and len(entry) >= 2 else (0, 0) + px, py = ( + (entry[0], entry[1]) + if isinstance(entry, list) and len(entry) >= 2 + else (0, 0) + ) at_x = point_x(px) - if at_x < plot["x"] - radius or \ - at_x > plot["x"] + plot["w"] + radius: + if ( + at_x < plot["x"] - radius + or at_x > plot["x"] + plot["w"] + radius + ): continue canvas.save() canvas.alpha(0.85) - canvas.ellipse_path(at_x, value_y(opt_num(py, 0)), radius, - radius) + canvas.ellipse_path(at_x, value_y(opt_num(py, 0)), radius, radius) canvas.paint(fill=color) canvas.restore() del index @@ -2515,13 +2963,17 @@ def point_x(value): for index, category in enumerate(categories): if index % stride: continue - self.chart_text(canvas, chart, plot["x"] + band * (index + 0.5), - plot["y"] + plot["h"] - + chart["xAxisLabel"]["size"] + 6, category, - parse_color(chart["xAxisLabel"]["color"], - (0.42, 0.45, 0.5, 1)), - chart["xAxisLabel"]["size"], "middle", - chart["xAxisLabel"]["weight"]) + self.chart_text( + canvas, + chart, + plot["x"] + band * (index + 0.5), + plot["y"] + plot["h"] + chart["xAxisLabel"]["size"] + 6, + category, + parse_color(chart["xAxisLabel"]["color"], (0.42, 0.45, 0.5, 1)), + chart["xAxisLabel"]["size"], + "middle", + chart["xAxisLabel"]["weight"], + ) if bars: group = band * 0.62 @@ -2529,46 +2981,62 @@ def point_x(value): for order, series in enumerate(bars): axis = axis_index(series) base = zero_y(axis) - color = self.series_color(chart, series, - chart["series"].index(series)) - item_style = series.get("itemStyle") \ - if isinstance(series.get("itemStyle"), dict) else {} + color = self.series_color(chart, series, chart["series"].index(series)) + item_style = ( + series.get("itemStyle") + if isinstance(series.get("itemStyle"), dict) + else {} + ) corner = item_style.get("borderRadius") - corner = opt_num(corner[0], 0) if isinstance(corner, list) \ + corner = ( + opt_num(corner[0], 0) + if isinstance(corner, list) else opt_num(corner, 0) + ) data = series.get("data") data = data if isinstance(data, list) else [] for index, raw in enumerate(data[:count]): value = opt_num(raw, 0) - left = plot["x"] + band * index + (band - group) / 2.0 \ - + slot * order + left = ( + plot["x"] + band * index + (band - group) / 2.0 + slot * order + ) top = value_y(value, axis) height = abs(base - top) - canvas.rect_path(left + 1, top if top <= base else base, - max(1.0, slot - 2), max(0.0, height), - min(corner, slot / 2.0)) + canvas.rect_path( + left + 1, + top if top <= base else base, + max(1.0, slot - 2), + max(0.0, height), + min(corner, slot / 2.0), + ) canvas.paint(fill=color) for series in lines: order = chart["series"].index(series) axis = axis_index(series) color = self.series_color(chart, series, order) - width = opt_num((series.get("lineStyle") or {}).get("width") - if isinstance(series.get("lineStyle"), dict) - else None, 2) + width = opt_num( + (series.get("lineStyle") or {}).get("width") + if isinstance(series.get("lineStyle"), dict) + else None, + 2, + ) data = series.get("data") data = data if isinstance(data, list) else [] values = [opt_num(value, 0) for value in data[:count]] - pts = [(plot["x"] + band * (index + 0.5), value_y(value, axis)) - for index, value in enumerate(values)] + pts = [ + (plot["x"] + band * (index + 0.5), value_y(value, axis)) + for index, value in enumerate(values) + ] if len(pts) < 2: continue path = self._line_path(pts, bool(series.get("smooth"))) area = series.get("areaStyle") if area is not None: canvas.save() - canvas.alpha(1.0 if isinstance(area, dict) - and area.get("color") else 0.25) + canvas.alpha( + 1.0 if isinstance(area, dict) and area.get("color") else 0.25 + ) self._emit(canvas, path) canvas.op("%s %s l" % (num(pts[-1][0]), num(zero_y(axis)))) canvas.op("%s %s l" % (num(pts[0][0]), num(zero_y(axis)))) @@ -2596,12 +3064,20 @@ def _line_path(self, pts, smooth): current = pts[index] following = pts[index + 1] after = pts[min(len(pts) - 1, index + 2)] - path.append(("c", - (current[0] + (following[0] - previous[0]) / 6.0, - current[1] + (following[1] - previous[1]) / 6.0), - (following[0] - (after[0] - current[0]) / 6.0, - following[1] - (after[1] - current[1]) / 6.0), - following)) + path.append( + ( + "c", + ( + current[0] + (following[0] - previous[0]) / 6.0, + current[1] + (following[1] - previous[1]) / 6.0, + ), + ( + following[0] - (after[0] - current[0]) / 6.0, + following[1] - (after[1] - current[1]) / 6.0, + ), + following, + ) + ) else: for point in pts[1:]: path.append(("l", point)) @@ -2615,8 +3091,10 @@ def _emit(self, canvas, path): canvas.op("%s %s l" % (num(segment[1][0]), num(segment[1][1]))) else: (a, b), (c, d), (e, f) = segment[1:] - canvas.op("%s %s %s %s %s %s c" - % (num(a), num(b), num(c), num(d), num(e), num(f))) + canvas.op( + "%s %s %s %s %s %s c" + % (num(a), num(b), num(c), num(d), num(e), num(f)) + ) # -- pie --------------------------------------------------------------- def draw_pie(self, canvas, chart): @@ -2626,8 +3104,12 @@ def draw_pie(self, canvas, chart): slices = [] for index, entry in enumerate(data): entry = entry if isinstance(entry, dict) else {} - slices.append((str(entry.get("name", index)), - max(0.0, opt_num(entry.get("value"), 0)))) + slices.append( + ( + str(entry.get("name", index)), + max(0.0, opt_num(entry.get("value"), 0)), + ) + ) total = sum(value for _, value in slices) or 1.0 plot = chart["grid"] cx = plot["x"] + plot["w"] / 2.0 @@ -2641,17 +3123,20 @@ def resolve(value): return float(value.strip()[:-1]) / 100.0 * limit return opt_num(value, 0) - inner, outer = resolve(pair[0]), resolve(pair[1] if len(pair) > 1 - else "70%") - item_style = series.get("itemStyle") \ - if isinstance(series.get("itemStyle"), dict) else {} + inner, outer = resolve(pair[0]), resolve(pair[1] if len(pair) > 1 else "70%") + item_style = ( + series.get("itemStyle") if isinstance(series.get("itemStyle"), dict) else {} + ) border = parse_color(item_style.get("borderColor"), (0, 0, 0, 0.0)) border_width = opt_num(item_style.get("borderWidth"), 0) label = series.get("label") formatter = None if label is not False: - formatter = label.get("formatter") if isinstance(label, dict) \ - and isinstance(label.get("formatter"), str) else "{b}" + formatter = ( + label.get("formatter") + if isinstance(label, dict) and isinstance(label.get("formatter"), str) + else "{b}" + ) angle = -math.pi / 2 for index, (name, value) in enumerate(slices): share = value / total @@ -2665,17 +3150,28 @@ def resolve(value): at_x = cx + math.cos(middle) * (outer + 12) at_y = cy + math.sin(middle) * (outer + 12) right = math.cos(middle) >= 0 - text = (formatter.replace("{b}", name) - .replace("{c}", format_number(value)) - .replace("{d}", format_number(round(share * 1000) / 10.0))) - canvas.line_path(cx + math.cos(middle) * outer, - cy + math.sin(middle) * outer, at_x, at_y) + text = ( + formatter.replace("{b}", name) + .replace("{c}", format_number(value)) + .replace("{d}", format_number(round(share * 1000) / 10.0)) + ) + canvas.line_path( + cx + math.cos(middle) * outer, + cy + math.sin(middle) * outer, + at_x, + at_y, + ) canvas.paint(stroke=color, width=1) - self.chart_text(canvas, chart, at_x + (4 if right else -4), - at_y + 4, text, - parse_color(chart["labelColor"], - (0.42, 0.45, 0.5, 1)), 12, - "start" if right else "end") + self.chart_text( + canvas, + chart, + at_x + (4 if right else -4), + at_y + 4, + text, + parse_color(chart["labelColor"], (0.42, 0.45, 0.5, 1)), + 12, + "start" if right else "end", + ) angle = end def _pie_slice(self, canvas, cx, cy, inner, outer, start, end): @@ -2686,26 +3182,38 @@ def arc(radius, from_angle, to_angle, move): k = 4.0 / 3.0 * math.tan(step / 4.0) angle = from_angle if move: - canvas.op("%s %s m" % (num(cx + math.cos(angle) * radius), - num(cy + math.sin(angle) * radius))) + canvas.op( + "%s %s m" + % ( + num(cx + math.cos(angle) * radius), + num(cy + math.sin(angle) * radius), + ) + ) for _ in range(steps): nxt = angle + step x1 = cx + math.cos(angle) * radius y1 = cy + math.sin(angle) * radius x2 = cx + math.cos(nxt) * radius y2 = cy + math.sin(nxt) * radius - canvas.op("%s %s %s %s %s %s c" - % (num(x1 - k * math.sin(angle) * radius), - num(y1 + k * math.cos(angle) * radius), - num(x2 + k * math.sin(nxt) * radius), - num(y2 - k * math.cos(nxt) * radius), - num(x2), num(y2))) + canvas.op( + "%s %s %s %s %s %s c" + % ( + num(x1 - k * math.sin(angle) * radius), + num(y1 + k * math.cos(angle) * radius), + num(x2 + k * math.sin(nxt) * radius), + num(y2 - k * math.cos(nxt) * radius), + num(x2), + num(y2), + ) + ) angle = nxt arc(outer, start, end, True) if inner > 0: - canvas.op("%s %s l" % (num(cx + math.cos(end) * inner), - num(cy + math.sin(end) * inner))) + canvas.op( + "%s %s l" + % (num(cx + math.cos(end) * inner), num(cy + math.sin(end) * inner)) + ) arc(inner, end, start, False) else: canvas.op("%s %s l" % (num(cx), num(cy))) @@ -2717,38 +3225,65 @@ def draw_legend(self, canvas, chart): pie = next((s for s in chart["series"] if s.get("type") == "pie"), None) if pie is not None: data = pie.get("data") if isinstance(pie.get("data"), list) else [] - entries = [(str((entry or {}).get("name", index)) - if isinstance(entry, dict) else str(index), - parse_color(chart["colors"][index % len(chart["colors"])])) - for index, entry in enumerate(data)] + entries = [ + ( + str((entry or {}).get("name", index)) + if isinstance(entry, dict) + else str(index), + parse_color(chart["colors"][index % len(chart["colors"])]), + ) + for index, entry in enumerate(data) + ] else: - entries = [(str(series.get("name") or "Series %d" % (index + 1)), - self.series_color(chart, series, index)) - for index, series in enumerate(chart["series"])] + entries = [ + ( + str(series.get("name") or "Series %d" % (index + 1)), + self.series_color(chart, series, index), + ) + for index, series in enumerate(chart["series"]) + ] if not entries: return face = face_name(chart["font"], is_bold(legend["weight"]), False) - widths = [legend["itemWidth"] + 8 - + text_width(self.encoder, name, face, legend["size"]) - for name, _ in entries] + widths = [ + legend["itemWidth"] + + 8 + + text_width(self.encoder, name, face, legend["size"]) + for name, _ in entries + ] span = sum(widths) + legend["itemGap"] * max(0, len(entries) - 1) left = max(8.0, (chart["w"] - span) / 2.0) height = max(legend["itemHeight"], legend["size"]) - top = legend["top"] if legend["top"] is not None \ + top = ( + legend["top"] + if legend["top"] is not None else chart["h"] - legend["bottom"] - height - 10 + ) swatch_y = top + (height - legend["itemHeight"]) / 2.0 baseline = top + height / 2.0 + legend["size"] * 0.35 for index, (name, color) in enumerate(entries): - canvas.rect_path(left, swatch_y, legend["itemWidth"], - legend["itemHeight"], - min(3.0, legend["itemHeight"] / 2.0)) + canvas.rect_path( + left, + swatch_y, + legend["itemWidth"], + legend["itemHeight"], + min(3.0, legend["itemHeight"] / 2.0), + ) canvas.paint(fill=color) - self.chart_text(canvas, chart, left + legend["itemWidth"] + 8, - baseline, name, - parse_color(legend["color"], (0.42, 0.45, 0.5, 1)), - legend["size"], "start", legend["weight"]) - left += widths[index] + (legend["itemGap"] - if index < len(entries) - 1 else 0) + self.chart_text( + canvas, + chart, + left + legend["itemWidth"] + 8, + baseline, + name, + parse_color(legend["color"], (0.42, 0.45, 0.5, 1)), + legend["size"], + "start", + legend["weight"], + ) + left += widths[index] + ( + legend["itemGap"] if index < len(entries) - 1 else 0 + ) class DeckRenderer(Renderer, ShapeMixin, TableMixin, ChartMixin): diff --git a/internal/clientconfig/builtin_skills/data-profiler/scripts/profile.py b/internal/clientconfig/builtin_skills/data-profiler/scripts/profile.py index 348da265..dc8e6eb6 100644 --- a/internal/clientconfig/builtin_skills/data-profiler/scripts/profile.py +++ b/internal/clientconfig/builtin_skills/data-profiler/scripts/profile.py @@ -144,7 +144,7 @@ def profile_delimited(path, delimiter, max_rows, top): if len(row) != len(cols): malformed.append(reader.line_num) continue - for stats, cell in zip(cols, row): + for stats, cell in zip(cols, row, strict=True): stats.add(cell) kind = "tsv" if delimiter == "\t" else "csv" report(path, kind, total, cols, malformed, top) diff --git a/internal/mcp/httptool.go b/internal/mcp/httptool.go index 7eb0a82a..e5bfec5a 100644 --- a/internal/mcp/httptool.go +++ b/internal/mcp/httptool.go @@ -251,6 +251,7 @@ func executeHTTPTool(ctx context.Context, client *http.Client, spec HTTPToolSpec // Multiple jq outputs are newline-joined; scalars/objects are rendered as compact // JSON. func applyResponseJQ(program string, body []byte) (out string, ok bool, err error) { + // nosemgrep: go.lang.security.deserialization.unsafe-deserialization-interface.go-unsafe-deserialization-interface -- interface{} is REQUIRED here, not a shortcut: the value is handed straight to a jq program, which operates on arbitrary JSON by definition. A concrete struct cannot express "whatever shape the response had". var input interface{} if jsonErr := json.Unmarshal(body, &input); jsonErr != nil { //nolint:nilerr // intentional: a non-JSON body is not an error — ok=false signals "pass the raw body through unfiltered" (response_jq applies only to JSON, per the issue spec). diff --git a/internal/mcp/testdata/dummy_server.py b/internal/mcp/testdata/dummy_server.py index cf0bf728..76785fe8 100644 --- a/internal/mcp/testdata/dummy_server.py +++ b/internal/mcp/testdata/dummy_server.py @@ -1,10 +1,10 @@ import sys import json -import os # Ensure unbuffered output sys.stdout.reconfigure(line_buffering=True) + def main(): while True: try: @@ -26,7 +26,7 @@ def main(): response["result"] = { "protocolVersion": "2024-11-05", "serverInfo": {"name": "dummy", "version": "1.0"}, - "capabilities": {} + "capabilities": {}, } elif method == "tools/list": response["result"] = { @@ -36,10 +36,8 @@ def main(): "description": "Echoes input", "inputSchema": { "type": "object", - "properties": { - "message": {"type": "string"} - } - } + "properties": {"message": {"type": "string"}}, + }, } ] } @@ -50,10 +48,7 @@ def main(): if tool_name == "echo": response["result"] = { "content": [ - { - "type": "text", - "text": f"Echo: {args.get('message')}" - } + {"type": "text", "text": f"Echo: {args.get('message')}"} ] } else: @@ -67,5 +62,6 @@ def main(): sys.stderr.write(str(e) + "\n") break + if __name__ == "__main__": main() diff --git a/internal/mcp/testdata/noisy_server.py b/internal/mcp/testdata/noisy_server.py index 5539452d..d7b7f642 100644 --- a/internal/mcp/testdata/noisy_server.py +++ b/internal/mcp/testdata/noisy_server.py @@ -22,7 +22,15 @@ def main() -> None: # 1. Stray non-JSON output (e.g. a library print). print("WARNING: something logged straight to stdout") # 2. Server-initiated notification. - print(json.dumps({"jsonrpc": "2.0", "method": "notifications/progress", "params": {"progress": 1}})) + print( + json.dumps( + { + "jsonrpc": "2.0", + "method": "notifications/progress", + "params": {"progress": 1}, + } + ) + ) # 3. Stale response to a request id that is not ours. print(json.dumps({"jsonrpc": "2.0", "id": 999999, "result": {"echoed": -1}})) # 4. The real response. diff --git a/internal/runner/runner.go b/internal/runner/runner.go index b7a2c286..4c9fc11d 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -25,7 +25,7 @@ import ( "errors" "fmt" "log" - "math/rand/v2" + "math/rand/v2" // nosemgrep: go.lang.security.audit.crypto.math_random.math-random-used -- used once, for +/-10% jitter on a retry interval (see rand.Int64N below). Nothing here is a secret, a token, or an identity; crypto/rand would only make backoff slower. "os" "strconv" "strings" diff --git a/internal/sandbox/fileops.py b/internal/sandbox/fileops.py index 8b48da85..ba77c41e 100644 --- a/internal/sandbox/fileops.py +++ b/internal/sandbox/fileops.py @@ -12,6 +12,7 @@ # full-file hashes, optional expected_sha256, no-op rejection, and a bounded # unified diff. Those semantics must never regress the #784 confinement. import base64 +import contextlib import difflib import errno import hashlib @@ -35,8 +36,14 @@ class StaleContent(Exception): def fail(kind, msg, **extra): - response = {"ok": False, "err_kind": kind, "err": msg, - "data_b64": "", "size": 0, "count": 0} + response = { + "ok": False, + "err_kind": kind, + "err": msg, + "data_b64": "", + "size": 0, + "count": 0, + } response.update(extra) return response @@ -74,17 +81,18 @@ def _open_dir_at(parent_fd, name, create): fd = os.open(name, flags, dir_fd=parent_fd) if created: # mkdir is umask-filtered; the file-tool contract is exact 0750. + # nosemgrep: python.lang.security.audit.insecure-file-permissions.insecure-file-permissions -- the rule advises 0o644, i.e. WORLD-READABLE, for a sandbox directory. Following it would be a security regression. 0750 is the file-tool contract and is deliberately tighter than the suggestion. os.fchmod(fd, 0o750) return fd except OSError as exc: if exc.errno in (errno.ELOOP, errno.ENOTDIR): - raise UnsafePath( - "directory component changed or is a symlink") from exc + raise UnsafePath("directory component changed or is a symlink") from exc raise except OSError as exc: if exc.errno in (errno.ELOOP, errno.ENOTDIR): raise UnsafePath( - "directory component is a symlink or not a directory") from exc + "directory component is a symlink or not a directory" + ) from exc raise @@ -150,9 +158,16 @@ def do_bind_root(req): try: root_fd = _open_root(req) info = os.fstat(root_fd) - return {"ok": True, "err_kind": "", "err": "", "data_b64": "", - "size": 0, "count": 0, "dev": info.st_dev, - "ino": info.st_ino} + return { + "ok": True, + "err_kind": "", + "err": "", + "data_b64": "", + "size": 0, + "count": 0, + "dev": info.st_dev, + "ino": info.st_ino, + } except UnsafePath as exc: return fail("unsafe_path", str(exc)) except OSError as exc: @@ -168,12 +183,18 @@ def _test_pause(req, parent_fd): if pause_ms <= 0: return name = req.get("test_ready_name", "") - if (not isinstance(name, str) or - not name.startswith(".fleet-fileop-test-") or - os.path.basename(name) != name): + if ( + not isinstance(name, str) + or not name.startswith(".fleet-fileop-test-") + or os.path.basename(name) != name + ): raise UnsafePath("invalid test rendezvous name") - fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | - os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=parent_fd) + fd = os.open( + name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + dir_fd=parent_fd, + ) os.close(fd) os.fsync(parent_fd) time.sleep(min(pause_ms, 10000) / 1000.0) @@ -181,8 +202,7 @@ def _test_pause(req, parent_fd): def _open_file(parent_fd, name): try: - fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, - dir_fd=parent_fd) + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=parent_fd) except FileNotFoundError: raise except IsADirectoryError: @@ -211,8 +231,11 @@ def _lstat_at(parent_fd, name): def _same_file(left, right): - return (left is not None and right is not None and - (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino)) + return ( + left is not None + and right is not None + and (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino) + ) def _sha256_open_at(parent_fd, name): @@ -226,9 +249,12 @@ def _sha256_open_at(parent_fd, name): break digest.update(chunk) after = os.fstat(fd) - if ((info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns) != - (after.st_dev, after.st_ino, after.st_size, - after.st_mtime_ns)): + if (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns) != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ): raise StaleContent("file changed while its hash was computed") return digest.hexdigest() finally: @@ -236,8 +262,7 @@ def _sha256_open_at(parent_fd, name): os.close(fd) -def _atomic_write(parent_fd, name, data, expected=None, - expected_digest=None): +def _atomic_write(parent_fd, name, data, expected=None, expected_digest=None): before = _lstat_at(parent_fd, name) if expected is not None and not _same_file(before, expected): raise UnsafePath("file changed while it was being edited") @@ -247,8 +272,12 @@ def _atomic_write(parent_fd, name, data, expected=None, tmp = ".fleet-fileop-%s" % secrets.token_hex(12) fd = -1 try: - fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | - os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=parent_fd) + fd = os.open( + tmp, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + dir_fd=parent_fd, + ) view = memoryview(data) while view: written = os.write(fd, view) @@ -256,16 +285,14 @@ def _atomic_write(parent_fd, name, data, expected=None, if owner is not None: current_owner = os.fstat(fd) if (current_owner.st_uid, current_owner.st_gid) != owner: - try: + # Best effort: preserving the destination's ownership on an + # overwrite is a nicety, not a safety property. The executor + # runs unprivileged and cannot chown to a foreign uid (e.g. + # a host-seeded workspace file mapping to container-root), so + # keep the executor-owned replacement rather than aborting a + # legitimate edit. + with contextlib.suppress(PermissionError): os.fchown(fd, owner[0], owner[1]) - except PermissionError: - # Best effort: preserving the destination's ownership on an - # overwrite is a nicety, not a safety property. The executor - # runs unprivileged and cannot chown to a foreign uid (e.g. - # a host-seeded workspace file mapping to container-root), - # so keep the executor-owned replacement rather than - # aborting a legitimate edit. - pass os.fchmod(fd, mode) os.fsync(fd) os.close(fd) @@ -282,17 +309,14 @@ def _atomic_write(parent_fd, name, data, expected=None, if expected_digest is not None: current_digest = _sha256_open_at(parent_fd, name) if current_digest != expected_digest: - raise StaleContent( - "file content changed before the edit was committed") + raise StaleContent("file content changed before the edit was committed") os.replace(tmp, name, src_dir_fd=parent_fd, dst_dir_fd=parent_fd) os.fsync(parent_fd) except BaseException: if fd >= 0: os.close(fd) - try: + with contextlib.suppress(OSError): os.unlink(tmp, dir_fd=parent_fd) - except OSError: - pass raise @@ -312,7 +336,7 @@ def _hash_and_window(fd, offset, limit): wanted_start = max(position, offset) wanted_end = chunk_end if end is None else min(chunk_end, end) if wanted_start < wanted_end: - chunks.append(chunk[wanted_start - position:wanted_end - position]) + chunks.append(chunk[wanted_start - position : wanted_end - position]) position = chunk_end return digest.hexdigest(), b"".join(chunks) @@ -328,14 +352,22 @@ def do_read(req): limit = int(req.get("limit", 0) or 0) digest, data = _hash_and_window(fd, offset, limit) after = os.fstat(fd) - if ((before.st_dev, before.st_ino, before.st_size, - before.st_mtime_ns) != - (after.st_dev, after.st_ino, after.st_size, - after.st_mtime_ns)): + if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ): raise StaleContent("file changed while it was being read") - return {"ok": True, "err_kind": "", "err": "", - "data_b64": base64.b64encode(data).decode("ascii"), - "size": before.st_size, "count": 0, "sha256": digest} + return { + "ok": True, + "err_kind": "", + "err": "", + "data_b64": base64.b64encode(data).decode("ascii"), + "size": before.st_size, + "count": 0, + "sha256": digest, + } except FileNotFoundError: return fail("not_found", "file not found") except IsADirectoryError: @@ -360,9 +392,15 @@ def do_write(req): _test_pause(req, parent_fd) data = base64.b64decode(req.get("data_b64", "")) _atomic_write(parent_fd, name, data) - return {"ok": True, "err_kind": "", "err": "", "data_b64": "", - "size": len(data), "count": 0, - "sha256": hashlib.sha256(data).hexdigest()} + return { + "ok": True, + "err_kind": "", + "err": "", + "data_b64": "", + "size": len(data), + "count": 0, + "sha256": hashlib.sha256(data).hexdigest(), + } except IsADirectoryError: return fail("is_dir", "path is a directory") except UnsafePath as exc: @@ -378,13 +416,21 @@ def _bounded_diff(path, before, after): old_lines = before.decode("utf-8", "replace").splitlines(keepends=True) new_lines = after.decode("utf-8", "replace").splitlines(keepends=True) name = os.path.basename(path) - lines = list(difflib.unified_diff( - old_lines, new_lines, fromfile=name + " (before)", - tofile=name + " (after)", n=3)) - added = sum(1 for line in lines - if line.startswith("+") and not line.startswith("+++")) - removed = sum(1 for line in lines - if line.startswith("-") and not line.startswith("---")) + lines = list( + difflib.unified_diff( + old_lines, + new_lines, + fromfile=name + " (before)", + tofile=name + " (after)", + n=3, + ) + ) + added = sum( + 1 for line in lines if line.startswith("+") and not line.startswith("+++") + ) + removed = sum( + 1 for line in lines if line.startswith("-") and not line.startswith("---") + ) text = "".join(lines) encoded = text.encode("utf-8") if len(encoded) > DIFF_MAX_BYTES: @@ -395,9 +441,11 @@ def _bounded_diff(path, before, after): def _stale(expected, current): return fail( - "stale", "file content has changed since it was last read " - "(expected sha256 %s, current %s); re-read the file and retry" % - (expected, current)) + "stale", + "file content has changed since it was last read " + "(expected sha256 %s, current %s); re-read the file and retry" + % (expected, current), + ) def do_edit(req): @@ -414,7 +462,7 @@ def do_edit(req): old_digest = hashlib.sha256(content).hexdigest() expected = (req.get("expected_sha256") or "").strip().lower() if expected.startswith("sha256:"): - expected = expected[len("sha256:"):] + expected = expected[len("sha256:") :] if expected and expected != old_digest: return _stale(expected, old_digest) @@ -423,20 +471,25 @@ def do_edit(req): count = content.count(old) if count == 0: response = fail("old_absent", "old_text not found in file") - if (b"\r\n" in content and old and - old.replace(b"\r\n", b"\n") in - content.replace(b"\r\n", b"\n")): + if ( + b"\r\n" in content + and old + and old.replace(b"\r\n", b"\n") in content.replace(b"\r\n", b"\n") + ): response["hint"] = ( "the file uses CRLF line endings — include \\r\\n in " "old_text exactly as view_file returned it, or re-read " - "the file") + "the file" + ) return response if count > 1 and not req.get("replace_all"): return fail( - "ambiguous", "old_text matches %d locations; edit_file " + "ambiguous", + "old_text matches %d locations; edit_file " "replaces exactly one — add surrounding context to make the " "match unique, or set replace_all=true" % count, - match_count=count) + match_count=count, + ) if req.get("replace_all"): updated = content.replace(old, new) @@ -445,12 +498,18 @@ def do_edit(req): count = 1 if updated == content: return fail( - "noop", "edit is a no-op (old_text and new_text produce " - "identical content)") + "noop", + "edit is a no-op (old_text and new_text produce identical content)", + ) try: - _atomic_write(parent_fd, name, updated, expected=info, - expected_digest=old_digest if expected else None) + _atomic_write( + parent_fd, + name, + updated, + expected=info, + expected_digest=old_digest if expected else None, + ) except StaleContent: try: current = _sha256_open_at(parent_fd, name) @@ -459,11 +518,20 @@ def do_edit(req): return _stale(expected, current) diff, added, removed = _bounded_diff(req["path"], content, updated) - return {"ok": True, "err_kind": "", "err": "", "data_b64": "", - "size": len(updated), "count": count, - "sha256": hashlib.sha256(updated).hexdigest(), - "old_sha256": old_digest, "match_count": count, - "added": added, "removed": removed, "diff": diff} + return { + "ok": True, + "err_kind": "", + "err": "", + "data_b64": "", + "size": len(updated), + "count": count, + "sha256": hashlib.sha256(updated).hexdigest(), + "old_sha256": old_digest, + "match_count": count, + "added": added, + "removed": removed, + "diff": diff, + } except FileNotFoundError: return fail("not_found", "file not found") except IsADirectoryError: @@ -485,8 +553,12 @@ def main(): except Exception as exc: print(json.dumps(fail("", "bad request: %s" % exc))) return - handler = {"read": do_read, "write": do_write, "edit": do_edit, - "bind_root": do_bind_root}.get(req.get("op")) + handler = { + "read": do_read, + "write": do_write, + "edit": do_edit, + "bind_root": do_bind_root, + }.get(req.get("op")) if handler is None: print(json.dumps(fail("", "unknown op: %r" % req.get("op")))) return diff --git a/internal/sched/handlers/elcano.go b/internal/sched/handlers/elcano.go index 440c498f..6b5d3dae 100644 --- a/internal/sched/handlers/elcano.go +++ b/internal/sched/handlers/elcano.go @@ -152,6 +152,7 @@ func (h *Handlers) ElcanoLogout(w http.ResponseWriter, r *http.Request) { // attributes must mirror how auth originally set the cookie for the browser // to actually clear it — forcing Secure here unconditionally would prevent // logout from clearing the cookie over plain HTTP. + // nosemgrep: go.lang.security.audit.net.cookie-missing-secure.cookie-missing-secure -- same reasoning as the G124 waiver: this is a DELETION cookie (Value="", MaxAge=-1) carrying no secret, and its attributes must mirror how auth set it or the browser will not clear it. Forcing Secure unconditionally would break logout over plain-HTTP dev. http.SetCookie(w, &http.Cookie{ //nolint:gosec // G124: deletion cookie (no secret); Secure is conditional on HTTPS so logout works over plain-HTTP dev, mirroring how the cookie was set — see comment above. Name: h.config.ElcanoCookieName, Value: "", diff --git a/internal/tools/python_bridge.py b/internal/tools/python_bridge.py index 77bca45c..86b12ed5 100644 --- a/internal/tools/python_bridge.py +++ b/internal/tools/python_bridge.py @@ -1,5 +1,6 @@ import atexit import base64 +import contextlib import binascii import datetime import json @@ -90,7 +91,9 @@ def write_figures(images, base_dir): try: # O_EXCL | O_NOFOLLOW: refuse to follow or overwrite a name a prior # cell's code might have pre-planted in the (writable) workspace. - fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + fd = os.open( + path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600 + ) try: os.write(fd, raw) finally: @@ -136,10 +139,8 @@ def reap_stale_kernels(): except OSError: continue if b"ipykernel_launcher" in cmdline: - try: + with contextlib.suppress(OSError): os.kill(pid, signal.SIGKILL) - except OSError: - pass def start_kernel(): @@ -162,30 +163,30 @@ def start_kernel(): # tempfile.mkstemp creates a unique empty file and returns (fd, path). # We close the fd immediately since ipykernel will overwrite the # content — we only needed the unique name. - fd, connection_file = tempfile.mkstemp(prefix=f"kernel-{os.getpid()}-", suffix=".json") + fd, connection_file = tempfile.mkstemp( + prefix=f"kernel-{os.getpid()}-", suffix=".json" + ) os.close(fd) # ipykernel creates the file itself with its own content. Our mkstemp # empty placeholder would make the while-loop below think the file # "exists" before the kernel has actually written to it, so we remove # our placeholder and let ipykernel create it fresh. - try: + with contextlib.suppress(OSError): os.unlink(connection_file) - except OSError: - pass # Start the kernel - cmd = [ - sys.executable, - "-m", "ipykernel_launcher", - "-f", connection_file - ] + cmd = [sys.executable, "-m", "ipykernel_launcher", "-f", connection_file] # Start process detached to avoid signal interference - kernel_process = subprocess.Popen( + # S603 waived on the Popen below: every element of cmd is internal — + # sys.executable, literal flags, and a connection-file path this process + # just created with mkstemp in the OS temp dir. Nothing model- or + # user-controlled reaches the argv. + kernel_process = subprocess.Popen( # noqa: S603 cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - start_new_session=True + start_new_session=True, ) # Wait for connection file to exist @@ -198,20 +199,20 @@ def start_kernel(): return connection_file + def cleanup(): """Kills the kernel and deletes the connection file.""" global kernel_process, connection_file if kernel_process: - try: + # Broad on purpose: the process group may already be gone (ProcessLookup), + # unkillable (Permission), or getpgid itself can fail on a reaped pid — + # cleanup must never raise past atexit. + with contextlib.suppress(Exception): os.killpg(os.getpgid(kernel_process.pid), signal.SIGTERM) - except Exception: - pass kernel_process = None if connection_file: - try: + with contextlib.suppress(OSError): os.unlink(connection_file) - except OSError: - pass connection_file = None @@ -305,46 +306,47 @@ def run_code_on_kernel(code, client, timeout_seconds=None): while time.time() < deadline: if not shell_reply_seen: try: - shell_msg = client.get_shell_msg(timeout=min(0.05, remaining_time(deadline))) + shell_msg = client.get_shell_msg( + timeout=min(0.05, remaining_time(deadline)) + ) if shell_msg["parent_header"].get("msg_id") == msg_id: shell_reply_seen = True shell_content = shell_msg.get("content", {}) if shell_content.get("status") == "error": traceback = shell_content.get("traceback", []) - error_content = strip_ansi("\n".join(traceback)) or error_content + error_content = ( + strip_ansi("\n".join(traceback)) or error_content + ) status = "error" except queue.Empty: pass try: # Get IOPub messages (streams, display_data, etc) - msg = client.get_iopub_msg(timeout=min(IOPUB_POLL_SECONDS, remaining_time(deadline))) - msg_type = msg['header']['msg_type'] - content = msg['content'] + msg = client.get_iopub_msg( + timeout=min(IOPUB_POLL_SECONDS, remaining_time(deadline)) + ) + msg_type = msg["header"]["msg_type"] + content = msg["content"] - if msg['parent_header'].get('msg_id') != msg_id: + if msg["parent_header"].get("msg_id") != msg_id: continue - if msg_type == 'stream': - if content['name'] == 'stdout': - stdout_content.append(content['text']) - elif content['name'] == 'stderr': - stderr_content.append(content['text']) - elif msg_type == 'execute_result': - data = content.get('data', {}) - result_content.append(data.get('text/plain', '')) - collect_image(data, images) - elif msg_type == 'display_data': - data = content.get('data', {}) - result_content.append(data.get('text/plain', '')) + if msg_type == "stream": + if content["name"] == "stdout": + stdout_content.append(content["text"]) + elif content["name"] == "stderr": + stderr_content.append(content["text"]) + elif msg_type in ("execute_result", "display_data"): + data = content.get("data", {}) + result_content.append(data.get("text/plain", "")) collect_image(data, images) - elif msg_type == 'error': - traceback = content.get('traceback', []) - error_content = strip_ansi('\n'.join(traceback)) + elif msg_type == "error": + traceback = content.get("traceback", []) + error_content = strip_ansi("\n".join(traceback)) status = "error" - elif msg_type == 'status': - if content['execution_state'] == 'idle': - idle_seen = True + elif msg_type == "status" and content["execution_state"] == "idle": + idle_seen = True except queue.Empty: pass @@ -378,6 +380,7 @@ def run_code_on_kernel(code, client, timeout_seconds=None): "images": images, } + # Patterns that indicate agent confusion about MCP tools MCP_IMPORT_PATTERNS = [ "from mcp", @@ -392,6 +395,7 @@ def run_code_on_kernel(code, client, timeout_seconds=None): "import internal.tools", ] + def check_mcp_confusion(code): """Check if code appears to be trying to import MCP tools incorrectly.""" code_lower = code.lower() @@ -429,32 +433,27 @@ def normalize_json_value(value): return None if str(value) in {"", "NaT"}: return None + # Duck-typed pandas/numpy interop below: each probe is best-effort and a + # third-party accessor can raise anything, so the suppressions are broad on + # purpose — a failed probe falls through to the next representation. to_python = getattr(value, "to_pydatetime", None) if callable(to_python): - try: + with contextlib.suppress(Exception): return normalize_json_value(to_python()) - except Exception: - pass item = getattr(value, "item", None) if callable(item): - try: + with contextlib.suppress(Exception): extracted = item() if extracted is not value: return normalize_json_value(extracted) - except Exception: - pass to_list = getattr(value, "tolist", None) if callable(to_list): - try: + with contextlib.suppress(Exception): return normalize_json_value(to_list()) - except Exception: - pass isoformat = getattr(value, "isoformat", None) if callable(isoformat): - try: + with contextlib.suppress(Exception): return isoformat() - except Exception: - pass return value @@ -466,7 +465,9 @@ def dump_json_line(value): _kernel_cwd = None # last cwd applied inside the kernel process -def execute_code(code, return_vars=None, timeout_seconds=None, workspace_dir=None, reset_kernel=False): +def execute_code( + code, return_vars=None, timeout_seconds=None, workspace_dir=None, reset_kernel=False +): """Executes code on the kernel and returns the result.""" global client, _kernel_cwd @@ -492,7 +493,7 @@ def execute_code(code, return_vars=None, timeout_seconds=None, workspace_dir=Non return { "status": "error", "output": f"Failed to start kernel: {str(e)}", - "error": str(e) + "error": str(e), } # Apply per-conversation workspace cwd INSIDE the kernel process. @@ -502,10 +503,7 @@ def execute_code(code, return_vars=None, timeout_seconds=None, workspace_dir=Non if workspace_dir and workspace_dir != _kernel_cwd: try: escaped = workspace_dir.replace("\\", "\\\\").replace("'", "\\'") - chdir_code = ( - "import os as _os\n" - f"_os.chdir('{escaped}')\n" - ) + chdir_code = f"import os as _os\n_os.chdir('{escaped}')\n" chdir_res = run_code_on_kernel(chdir_code, client) if chdir_res["status"] == "success": _kernel_cwd = workspace_dir @@ -594,16 +592,15 @@ def execute_code(code, return_vars=None, timeout_seconds=None, workspace_dir=Non var_res = run_code_on_kernel(extract_script, client) if var_res["status"] == "success": - try: - with open(tmp_vars_file, "r", encoding="utf-8") as f: - vars_data = json.load(f) - except Exception: - pass # malformed/missing file → keep vars_data empty + # malformed/missing file → keep vars_data empty + with ( + contextlib.suppress(Exception), + open(tmp_vars_file, "r", encoding="utf-8") as f, + ): + vars_data = json.load(f) finally: - try: + with contextlib.suppress(OSError): os.unlink(tmp_vars_file) - except OSError: - pass # Consolidate legacy output field for backward compatibility final_output = "" @@ -629,7 +626,7 @@ def execute_code(code, return_vars=None, timeout_seconds=None, workspace_dir=Non return { "status": res["status"], - "output": final_output.strip(), # Legacy field + "output": final_output.strip(), # Legacy field "stdout": stdout_with_warning, "stderr": res["stderr"], "vars": normalize_json_value(vars_data), @@ -642,9 +639,10 @@ def execute_code(code, return_vars=None, timeout_seconds=None, workspace_dir=Non return { "status": "error", "output": f"Execution error: {str(e)}", - "error": str(e) + "error": str(e), } + def main(): # Cleanup on SIGTERM/SIGINT (Go side's terminateBridge) AND on normal # exit (SystemExit, unhandled EOF on stdin). atexit fires even for @@ -681,18 +679,28 @@ def main(): except OSError as e: sys.stderr.write(f"workspace_dir chdir failed: {e}\n") - result = execute_code(code, return_vars, timeout_seconds=timeout_seconds, workspace_dir=workspace_dir, reset_kernel=reset_kernel) + result = execute_code( + code, + return_vars, + timeout_seconds=timeout_seconds, + workspace_dir=workspace_dir, + reset_kernel=reset_kernel, + ) # Print result as JSON on one line print(dump_json_line(result), flush=True) except json.JSONDecodeError: - print(dump_json_line({"status": "error", "output": "Invalid JSON input"}), flush=True) + print( + dump_json_line({"status": "error", "output": "Invalid JSON input"}), + flush=True, + ) except KeyboardInterrupt: pass finally: cleanup() + if __name__ == "__main__": main() diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..ed18d40f --- /dev/null +++ b/ruff.toml @@ -0,0 +1,81 @@ +# ruff — the Python lane of the lint gate. +# +# WHY THIS EXISTS: fleet has 13 Python files (the sandbox FileOp helper, the +# python bridge, the bento-slides and data-profiler skill scripts, MCP test +# servers, icon/doc generators) and until now NOTHING linted them. Go had +# golangci-lint and the web tier had oxlint; Python had nothing, so its only +# coverage was whatever CodeQL's code-quality suite happened to notice. +# +# That is the wrong tool for the job in both directions. CodeQL took ~40s to +# report 28 note-level Python issues with no autofix, while ruff finds the same +# class of thing in well under a second and fixes much of it. So the Python +# quality lane is ruff's, and CodeQL keeps only its security queries (see +# docs/CODEQL.md). +# +# RULE SELECTION IS DELIBERATELY NARROW, and the numbers behind that are worth +# recording so nobody widens it by accident. Measured on this tree: +# +# default (E4,E7,E9,F) -> 3 findings <- what we gate on +# E,F,W,I,UP,B,SIM,ISC,PLR,PLW,S -> 333 findings +# +# The 330-finding difference is almost entirely style churn, not defects: +# 176 UP031 (%-format vs f-string), 43 PLR2004 (magic values), 35 E501 (line +# length). Gating on that would mean a 300-commit reformat for no correctness +# gain, so it is out of scope here rather than silently enabled. +# +# One rule was considered and deliberately REJECTED, because its only findings +# in this tree are correct code: +# +# PLR0124 (name compared with itself) — its 3 hits in bento_pdf.py are the +# idiomatic NaN test (`value != value` is true only for NaN). CodeQL's +# py/comparison-of-identical-expressions flagged the same 3. Enabling it +# would mean three `# noqa` comments on correct code. +# +# B, SIM and S started here as measured-but-deferred (21 findings) and were then +# fixed and ENABLED rather than left as a documented backlog: the two zip() +# sites got strict=True (both provably equal-length — one appends to both lists +# in lockstep, the other is behind an explicit len(row) != len(cols) guard), the +# unclosed NamedTemporaryFile was restructured into its with-block, the +# deliberate best-effort try/except-pass sites became explicit +# contextlib.suppress with the intent stated at each, and the one subprocess +# call carries a reasoned `# noqa: S603` (argv is sys.executable plus internal +# literals; nothing model- or user-controlled). The bandit tier (S) firing on a +# NEW line is therefore a real question to answer, not pre-existing noise. +# +# The gate is therefore "clean today, and stays clean": three real findings were +# fixed to get here (an unused import, a byte-identical duplicate function +# definition, and a lambda assignment), so a NEW default-rule violation is a +# real regression rather than noise in a backlog. +# +# FORMATTING IS ALSO GATED (`ruff format --check`, in CI and `make lint`). The +# whole tree was ruff-formatted in one commit — a 9-file, ~3.7k-line diff kept +# separate from any behavioural change — and validated against the full Go test +# suite (the bento/fileops golden tests exercise these scripts). From here a +# format failure means one new file, fixed by running `ruff format .`. + +# Match the lowest Python the sandbox image and the skill scripts must run on. +# Declared here so the rules that are version-sensitive (pyupgrade et al, if +# ever enabled) have one declaration point rather than a guess. +target-version = "py311" + +line-length = 88 + +exclude = [ + "node_modules", + "web/.next", + ".git", +] + +[lint] +# The default rule set: pycodestyle errors (E4 imports, E7 statements, +# E9 syntax/IO) plus Pyflakes (F — undefined names, unused imports, redefined +# names, unused locals). This is the "is it actually broken" tier; see the +# header for why the style tiers are not enabled. +select = ["E4", "E7", "E9", "F", "B", "SIM", "S"] + +[lint.per-file-ignores] +# MCP test servers and testdata fixtures are deliberately minimal stand-ins — +# they exist to be spawned and to misbehave in specific ways, so an unused +# import or an odd construct there can be the point of the fixture. +"internal/mcp/testdata/*.py" = ["F401"] +"cmd/fleet/testdata/*.py" = ["F401"] diff --git a/scripts/check-grype-policy.sh b/scripts/check-grype-policy.sh index 342dd637..b30ce8aa 100755 --- a/scripts/check-grype-policy.sh +++ b/scripts/check-grype-policy.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash -# Fail only for actionable CRITICAL vulnerabilities in Fedora RPMs. +# Fail for actionable CRITICAL and HIGH vulnerabilities in Fedora RPMs. +# +# High was added to the gate after measuring, not before: the published +# sandbox image at the time of the change carried zero fixable Critical or +# High RPM findings (its only fixable findings were two Medium openssh +# advisories), so the tightened gate started clean rather than arming over a +# backlog. Medium and below stay report-only — the image tracks +# fedora-minimal:latest, so routine rebuilds pick those up without a gate. # # Grype also catalogs Python dist-info shipped *by* Fedora RPMs as independent # PyPI packages. Those records use upstream versions/advisories and can claim a @@ -19,19 +26,19 @@ command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 2; } filter='[ .matches[] - | select((.vulnerability.severity // "" | ascii_downcase) == "critical") + | select((.vulnerability.severity // "" | ascii_downcase) as $s | $s == "critical" or $s == "high") | select((.artifact.type // "") == "rpm") | select(((.vulnerability.fix.versions // []) | length) > 0) ] | unique_by([.vulnerability.id, .artifact.name, .artifact.version])' findings="$(jq -r "$filter"'[] | [.vulnerability.id, .artifact.name, .artifact.version, (.vulnerability.fix.versions | join(","))] | @tsv' "$report")" if [[ -z "$findings" ]]; then - echo "Grype policy: no fixable CRITICAL Fedora RPM findings." + echo "Grype policy: no fixable CRITICAL or HIGH Fedora RPM findings." exit 0 fi count="$(awk 'NF { count++ } END { print count + 0 }' <<<"$findings")" -echo "Grype policy: $count fixable CRITICAL Fedora RPM finding(s):" >&2 +echo "Grype policy: $count fixable CRITICAL/HIGH Fedora RPM finding(s):" >&2 awk -F $'\t' '{ printf " %s %s %s fix: %s\n", $1, $2, $3, $4 }' <<<"$findings" >&2 echo "Rebuild against Fedora latest or update the affected RPM; do not shadow it with an ad-hoc language-package pin." >&2 exit 1 diff --git a/scripts/check-npm-overrides.sh b/scripts/check-npm-overrides.sh new file mode 100755 index 00000000..81fb4cd7 --- /dev/null +++ b/scripts/check-npm-overrides.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# check-npm-overrides.sh — fail when a security override has become droppable. +# +# scripts/rampart-service/package.json carries two `overrides` that force +# patched versions of transitive dependencies whose parents have not released a +# fix yet: +# +# sharp ^0.35.3 — @huggingface/transformers pins sharp ^0.34.5, which +# carries the libvips CVEs (CVE-2026-33327/-33328/-35590/ +# -35591, GHSA-f88m-g3jw-g9cj). +# adm-zip ^0.6.0 — onnxruntime-node pins adm-zip ^0.5.16, which carries +# GHSA-xcpc-8h2w-3j85 (crafted-ZIP 4 GB allocation). +# +# An override is a fork of upstream's intent: correct while upstream is broken, +# and pure drift the day upstream fixes itself — at which point Dependabot's +# normal updates are silently pinned down by us instead. Nothing else notices +# that day. This check does: it asks the registry what floor each PARENT now +# declares, and FAILS with removal instructions once the parent's own range +# reaches the patched line. So the reminder to drop the override is a red build +# with a two-line fix, not a stale-pin archaeology session years later. +# +# Registry unreachable / output unparsable is a SKIP with a notice, never a +# failure: this check's job is "tell me when the override is droppable", and a +# network flake is not evidence of that. The vulnerability gate itself is +# `npm audit` in the same CI job, which does fail closed on its own findings. +set -uo pipefail + +# floor RANGE -> x.y.z: the minimum version of a caret/tilde/plain range. +# The parents' published ranges are simple ("^0.34.5"); anything fancier +# parses to "unknown" and is treated as a skip, not a verdict. +floor() { + local r="${1#\^}"; r="${r#~}"; r="${r#>=}" + if [[ "$r" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+) ]]; then + printf '%s.%s.%s' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" + else + printf 'unknown' + fi +} + +# ge A B — true when version A >= B (numeric, three components). +ge() { + local IFS=. a b + read -ra a <<<"$1"; read -ra b <<<"$2" + for i in 0 1 2; do + if (( ${a[$i]:-0} > ${b[$i]:-0} )); then return 0; fi + if (( ${a[$i]:-0} < ${b[$i]:-0} )); then return 1; fi + done + return 0 +} + +# check PARENT DEP PATCHED_FLOOR — fail if PARENT's declared range for DEP now +# starts at or above PATCHED_FLOOR (the override is then droppable). +stale=0 +check() { + local parent="$1" dep="$2" patched="$3" range f + range="$(npm view "$parent@latest" "dependencies.$dep" 2>/dev/null || true)" + if [ -z "$range" ]; then + echo "notice: could not read $parent's $dep range from the registry — skipping (not a verdict)." + return 0 + fi + f="$(floor "$range")" + if [ "$f" = "unknown" ]; then + echo "notice: $parent declares $dep '$range' — cannot parse a floor, skipping." + return 0 + fi + if ge "$f" "$patched"; then + echo "::error::$parent@latest now declares $dep '$range' (floor $f >= $patched):" + echo " the '$dep' override in scripts/rampart-service/package.json is DROPPABLE." + echo " Remove it, regenerate package-lock.json (npm install --package-lock-only)," + echo " and re-run npm audit — upstream now ships the patched line itself." + stale=1 + else + echo "override for $dep still required: $parent@latest pins '$range' (floor $f < $patched)." + fi +} + +check "@huggingface/transformers" "sharp" "0.35.0" +check "onnxruntime-node" "adm-zip" "0.6.0" + +exit "$stale" diff --git a/scripts/check_versions_test.go b/scripts/check_versions_test.go index dbff4c51..56a90312 100644 --- a/scripts/check_versions_test.go +++ b/scripts/check_versions_test.go @@ -172,7 +172,8 @@ func TestDuplicatedToolPinsAgree(t *testing.T) { {"GRYPE_VERSION", ".github/workflows/grype-scheduled.yml", regexp.MustCompile(`GRYPE_VERSION:\s*'([^']+)'`)}, {"GRYPE_SHA256", ".github/workflows/grype-scheduled.yml", regexp.MustCompile(`GRYPE_SHA256:\s*'([^']+)'`)}, {"GITLEAKS_VERSION", ".github/workflows/dev-ci.yml", regexp.MustCompile(`GITLEAKS_VERSION:\s*'([^']+)'`)}, - {"golangci-lint version", ".github/workflows/dev-ci.yml", regexp.MustCompile(`golangci-lint-action@v\d+\s+with:\s+(?:#[^\n]*\n\s+)*version:\s*(v[\d.]+)`)}, + {"RUFF_VERSION", ".github/workflows/dev-ci.yml", regexp.MustCompile(`RUFF_VERSION:\s*'([^']+)'`)}, + {"golangci-lint version", ".github/workflows/dev-ci.yml", regexp.MustCompile(`golangci-lint-action@\S+[^\n]*\n\s*with:\s+(?:#[^\n]*\n\s+)*version:\s*(v[\d.]+)`)}, } { a := tc.re.FindStringSubmatch(ci) b := tc.re.FindStringSubmatch(readFile(t, root, tc.other)) @@ -314,7 +315,7 @@ func TestGoMinorAgreesEverywhere(t *testing.T) { func TestGolangciLintPinAgreesWithDocs(t *testing.T) { root := repoRoot(t) - pin := regexp.MustCompile(`golangci-lint-action@v\d+\s+with:\s+(?:#[^\n]*\n\s+)*version:\s*(v[\d.]+)`). + pin := regexp.MustCompile(`golangci-lint-action@\S+[^\n]*\n\s*with:\s+(?:#[^\n]*\n\s+)*version:\s*(v[\d.]+)`). FindStringSubmatch(readFile(t, root, ".github/workflows/ci.yml")) if pin == nil { t.Fatal("ci.yml: could not find the golangci-lint-action version pin") diff --git a/scripts/generate-icons.py b/scripts/generate-icons.py index 8199a0a7..57de4d06 100755 --- a/scripts/generate-icons.py +++ b/scripts/generate-icons.py @@ -8,6 +8,7 @@ Outputs: web/src/app/ favicon.ico, icon.svg, apple-icon.png web/public/app-icons/ favicon-16/32, icon-192/512, maskable-icon-512 """ + import io from pathlib import Path diff --git a/scripts/rampart-service/package-lock.json b/scripts/rampart-service/package-lock.json new file mode 100644 index 00000000..5262df8b --- /dev/null +++ b/scripts/rampart-service/package-lock.json @@ -0,0 +1,1042 @@ +{ + "name": "fleet-rampart-service", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "fleet-rampart-service", + "version": "1.0.0", + "dependencies": { + "@huggingface/transformers": "^4.2.0", + "@nationaldesignstudio/rampart": "^0.1.3" + }, + "engines": { + "node": ">=24" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@huggingface/jinja": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.9.tgz", + "integrity": "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tokenizers": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.1.3.tgz", + "integrity": "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==", + "license": "Apache-2.0" + }, + "node_modules/@huggingface/transformers": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.2.0.tgz", + "integrity": "sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.6", + "@huggingface/tokenizers": "^0.1.3", + "onnxruntime-node": "1.24.3", + "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", + "sharp": "^0.34.5" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.3.tgz", + "integrity": "sha512-RMnFX7YQsMoh7lWfcM4NEHHymBX/rLuKNPVM84XE9ONPcaSCDgE7CHIHpSgPcO2xcRthgBy1HfNO319mwhIAkg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.2" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.3.tgz", + "integrity": "sha512-Xo+5uFBtLN0BKqieTxiFzFPQAUlBbbH5iBKyRX/z1JrbnYsHTfKJnUfL8+p2TPXr1pXqao4eeL4Rl144uDpK9w==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.2" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.3.tgz", + "integrity": "sha512-lUxcqWIj2wMQ9BrwNjngcr1gWUr5xgaGThBRqPPalIC2n67Cqj1uPh8NnA/ZhAg8hUbKl+kVHKwgUIwe6ZYPrg==", + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.2.tgz", + "integrity": "sha512-9J6ypZFpQBj4YnePGoq/S38w6nz+vqg5WZLrLGY4YuSemdMq47GMLBPO42MzwdGwpg/agZ7xzZcFHa48xlywfg==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.2.tgz", + "integrity": "sha512-m2pW1n6cns9VaubNwsZ+c3CRYjxNQWgJ5gPlnL1nbBcpkBvFm6SCFN5o0psFHI8w9n11NKhFkeEDns98tiqbEw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.2.tgz", + "integrity": "sha512-1eMLzy92I4J6rmi4mAT8yC3HxOtniyGELlzGbNMLLeqe052ahFQ0h6LFq+lh5DsDIdYViIDst08abvSbcEdLXQ==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.2.tgz", + "integrity": "sha512-dqVSFynCox4C/J8kT16V7SIFAns0IjgLwkvYT7p8LQVmJ5OS5b6tI9IGflxTeuBS//zXeFIUbwt5dwxyZ17cnA==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.2.tgz", + "integrity": "sha512-3z0NHDxD6n5I9gc05U1eW1AyRm+Gznzq3naMrthPNqE6oYykcogW0l/jfpJdjYnuNl8R7yI9pNbE1XiUeyq0Aw==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.2.tgz", + "integrity": "sha512-bsb4rI+NldGOsXuej2r8OdSS8+zXDVaCWxyWrcv6kneTOlgAHtZABRzBBCwdsPiD90J4myNJuHpg6kA20ImW/w==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.2.tgz", + "integrity": "sha512-/ABshyj8gCpyIrNXnHn4LorDJ0HHm1VhXPBlxZ8zAtfVPAaSafXPGn+sUSIRiwaSBy0mmFjSjiXI5mkcwdChKQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.2.tgz", + "integrity": "sha512-ITPEtgffGJ0S6G9dRyw/366tJQqFRcHWPHhC+Stpg3Z8AEMrDrTr2lhdz4f/Y/HMbRh//7Z5mBzEpVdi62Oc3w==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.2.tgz", + "integrity": "sha512-zE9EdiUzUmg5mDT5a1rk5fYJ6GWPloTwWBYDS14naqHsL+EaMpDj1AWnpLgh3u0YCORv2Tt50wrcrpYqkP97Kw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.2.tgz", + "integrity": "sha512-m0lrLiUt+lBYnCFr8qV/65yMR4E/c7/wf78I5eKTdkEakFAlZ9QlzEM3QIhhAwVeUhLAHLcCq7a7Vszq/oFNZQ==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.3.tgz", + "integrity": "sha512-affVWCTLooy8TSxbDx2qkzuDeaWLNVBA+P//FNBirHsXpP2fuBhk5AuboYUnrDnzoXes8GFjpTx0SBFOCRg+FA==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.2" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.3.tgz", + "integrity": "sha512-QgKDspHPnrU+GQ55XPhGwyhC8acLVOOSyAvo1oVfFmrIXLkDNmGWzAfDZ4xK8oSA1qBQrALcHX0G5UZni/SuFQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.2" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.3.tgz", + "integrity": "sha512-sMd8rDxmpLOwv/7N44klFjOD5DUO7FLdjiXDI0hoxYaf7Ar262dQIEkosE98bps+5HPLtp/EvNqeqQtOycP/IA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.2" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.3.tgz", + "integrity": "sha512-0Eob78yjlYPfL5vMNWAW55l3R9Y6BQS/gOfe0ZcP9mEz9ohhKSt4im1hayiknXgf8AWrFqMvJcKIdmLmEe7yeQ==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.2" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.3.tgz", + "integrity": "sha512-KgAxQ0DxpNOq1rG2t5cgTgShJFGSuU7XO45cqC+1NVOuZnP6tlgZRuSYOfNupGkHID0o3cJOsw4DVeJpMovcGw==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.2" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.3.tgz", + "integrity": "sha512-8pqvxubL2PGdhlPy6GLqzDYMUjyRmKAwKHYKixpdJYBUK7PJ0C029XdsnpFIdgRZG68fZiGdHVWcKPvtiPB4cA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.2" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.3.tgz", + "integrity": "sha512-Vz0iQjzzcSX3HCbfwFfCSG/9SCIqyO0mH2sXyiHaAYfBk0cRsCWXRyQYX0ovCK/PAQBbTzQ0dsPQHh5MAFL59w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.2" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.3.tgz", + "integrity": "sha512-6O1NPKcDVj9QEdg7Hx549EX8U0rp6yXQERqru6yRN7fGBn32UvIRJUlWnk+8xDCiG76hXVBbX82NZ/ZKr0euIg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.2" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.3.tgz", + "integrity": "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w==", + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.1" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.3.tgz", + "integrity": "sha512-2rnq7bX3NzeR2T4YWgz8qiG4h3TSdMe+vN1iQXpJleSJ3SM5zQ8Fy2SyyXAWlbxpEZ2Y+Z4u1BePgJEYbSy80Q==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.3.tgz", + "integrity": "sha512-4bPwFdMbeC4JQ8L8LOyWp6nsHcboP5fxkp6iPOXz2Vg49R42TuMs2whkJ5OAP4/Ul035qOzy0AecOF9VOscn4w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.3.tgz", + "integrity": "sha512-r53mXsBN6lFUDiST764SvgwUdHAqM4rPAiDzAmf4fLoB6X/rkfyTrLCg6+g17wJJiCmB3JYgHuUldCWUIRFSXw==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.3.tgz", + "integrity": "sha512-D4y1vNeZrIIJCN+uHaWVtH86B+aCrdMYYjicy9pXHvbGZeGYLLSd3wdVuC37FxVXlU1ARsk84eKWfWMXGYEqvA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@nationaldesignstudio/rampart": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@nationaldesignstudio/rampart/-/rampart-0.1.3.tgz", + "integrity": "sha512-N3AMnPO1nGxUfMhTm+zJTuhXRIcdR/ETMIMyVr741pgxQcaRNMJVhKgN2nfEmwnXwGsViWpxFQOO0YYs8zZR2w==", + "license": "CC-BY-4.0", + "peerDependencies": { + "@huggingface/transformers": ">=3" + }, + "peerDependenciesMeta": { + "@huggingface/transformers": { + "optional": true + } + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, + "node_modules/@types/node": { + "version": "26.2.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", + "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/adm-zip": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.0.tgz", + "integrity": "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==", + "license": "MIT", + "engines": { + "node": ">=14.0" + } + }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT" + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "license": "MIT", + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/onnxruntime-common": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz", + "integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz", + "integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.5.16", + "global-agent": "^3.0.0", + "onnxruntime-common": "1.24.3" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.26.0-dev.20260416-b7804b056c", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.26.0-dev.20260416-b7804b056c.tgz", + "integrity": "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.24.0-dev.20251116-b39e144322", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.0-dev.20251116-b39e144322.tgz", + "integrity": "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==", + "license": "MIT" + }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, + "node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "license": "MIT" + }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/sharp": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.3.tgz", + "integrity": "sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q==", + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.3", + "@img/sharp-darwin-x64": "0.35.3", + "@img/sharp-freebsd-wasm32": "0.35.3", + "@img/sharp-libvips-darwin-arm64": "1.3.2", + "@img/sharp-libvips-darwin-x64": "1.3.2", + "@img/sharp-libvips-linux-arm": "1.3.2", + "@img/sharp-libvips-linux-arm64": "1.3.2", + "@img/sharp-libvips-linux-ppc64": "1.3.2", + "@img/sharp-libvips-linux-riscv64": "1.3.2", + "@img/sharp-libvips-linux-s390x": "1.3.2", + "@img/sharp-libvips-linux-x64": "1.3.2", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.2", + "@img/sharp-libvips-linuxmusl-x64": "1.3.2", + "@img/sharp-linux-arm": "0.35.3", + "@img/sharp-linux-arm64": "0.35.3", + "@img/sharp-linux-ppc64": "0.35.3", + "@img/sharp-linux-riscv64": "0.35.3", + "@img/sharp-linux-s390x": "0.35.3", + "@img/sharp-linux-x64": "0.35.3", + "@img/sharp-linuxmusl-arm64": "0.35.3", + "@img/sharp-linuxmusl-x64": "0.35.3", + "@img/sharp-webcontainers-wasm32": "0.35.3", + "@img/sharp-win32-arm64": "0.35.3", + "@img/sharp-win32-ia32": "0.35.3", + "@img/sharp-win32-x64": "0.35.3" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, + "node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "license": "MIT" + } + } +} diff --git a/scripts/rampart-service/package.json b/scripts/rampart-service/package.json index b514fa98..0e8db83a 100644 --- a/scripts/rampart-service/package.json +++ b/scripts/rampart-service/package.json @@ -14,5 +14,9 @@ }, "engines": { "node": ">=24" + }, + "overrides": { + "sharp": "^0.35.3", + "adm-zip": "^0.6.0" } } diff --git a/web/e2e/live/fixtures.ts b/web/e2e/live/fixtures.ts index 663d509b..8483f37a 100644 --- a/web/e2e/live/fixtures.ts +++ b/web/e2e/live/fixtures.ts @@ -1,4 +1,5 @@ import { test as base, expect, request } from "@playwright/test"; +import type { BrowserContext } from "@playwright/test"; export { expect, request }; @@ -17,7 +18,7 @@ const SCHED_USERNAME = process.env.E2E_SCHED_USERNAME ?? "e2e"; export const creds = { email: TEST_EMAIL, password: TEST_PASSWORD, schedUsername: SCHED_USERNAME }; -type AuthCookies = Parameters[0]; +type AuthCookies = Parameters[0]; // wipeConversations deletes every conversation for the logged-in user via the // real DELETE endpoint, so each test starts from a clean slate (conversations diff --git a/web/src/proxy.ts b/web/src/proxy.ts index a37a5873..23afb92d 100644 --- a/web/src/proxy.ts +++ b/web/src/proxy.ts @@ -96,6 +96,7 @@ function decorate(res: NextResponse, pathname: string): NextResponse { res.headers.set(BUILD_ID_HEADER, currentBuildId()); res.headers.set("Cache-Control", "no-store, must-revalidate"); res.headers.set("Content-Security-Policy", contentSecurityPolicy(pathname)); + // nosemgrep: javascript.express.security.x-frame-options-misconfiguration.x-frame-options-misconfiguration -- the value is the literal string "DENY". No user input reaches this header; the rule fires on the shape of the call, not on a real taint path. res.headers.set("X-Frame-Options", "DENY"); res.headers.set("X-Content-Type-Options", "nosniff"); res.headers.set("Referrer-Policy", "strict-origin-when-cross-origin");