diff --git a/.allowed_signers b/.allowed_signers new file mode 100644 index 0000000..a87e50e --- /dev/null +++ b/.allowed_signers @@ -0,0 +1 @@ +spam_blackhole@farcloser.world namespaces="git" sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIACDOXbkl7DthgVLTVZr8TNQcyQUX00MAqB3mWHik/vhAAAABHNzaDo= apostasie@farcloser.world diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..bd078af --- /dev/null +++ b/.editorconfig @@ -0,0 +1,57 @@ +# DO NOT EDIT MANUALLY. +# This file is common to all projects and managed by limen. +# Global configuration changes proposals can be discussed on https://github.com/farcloser/limen + +# https://editorconfig.org +root = true + +[*] +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +indent_style = space +indent_size = 2 + +# --- whitespace-significant formats --- +[*.{diff,patch}] +trim_trailing_whitespace = false + +[*.md] +trim_trailing_whitespace = false + +# --- Go: tabs, rendered 4 wide (gofmt emits tabs; width is display-only) --- +[*.go] +indent_style = tab +indent_size = 4 + +# --- task runner / build --- +[{Justfile,justfile,.justfile}] +indent_size = 4 + +[*.just] +indent_size = 4 + +[Makefile] +indent_style = tab + +# --- data formats (YAML must be spaces, never tabs) --- +[*.{json,jsonc,yaml,yml,toml}] +indent_size = 2 + +# --- JavaScript / TypeScript (2-space matches Prettier/Biome defaults) --- +[*.{js,cjs,mjs,jsx,ts,cts,mts,tsx}] +indent_size = 2 + +# --- CSS family --- +[*.{css,scss,sass,less,pcss}] +indent_size = 2 + +# --- HTML / templates --- +[*.{html,htm,vue,svelte,astro}] +indent_size = 2 + +# --- Rust: rustfmt is 4-space, 100-col --- +[*.rs] +indent_size = 4 +max_line_length = 100 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..68eac05 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,10 @@ +# Treat all files as binary, with no git magic updating line endings. +# This produces predictable results in different environments. +# +# Contributors on Windows will need to use a modern version of git +# and editors capable of LF line endings. +# +# See https://github.com/golangci/golangci-lint/issues/580 +# See https://github.com/golang/go/blob/master/.gitattributes + +* -text diff --git a/.github/actions/setup-aqua/action.yaml b/.github/actions/setup-aqua/action.yaml new file mode 100644 index 0000000..d818972 --- /dev/null +++ b/.github/actions/setup-aqua/action.yaml @@ -0,0 +1,54 @@ +name: Setup aqua +description: > + Install aqua (pinned, checksum-verified), put its bin directory on PATH, and authorize the repository's committed policy. aqua itself is the one tool aqua cannot pin. Same pins as limen-install: the installer script is fetched at an exact tag and checksum-verified, and it installs an exact aqua version. +runs: + using: composite + steps: + - name: Install aqua (pinned, checksum-verified) + shell: bash + env: + # Bumped manually, as a pair: Renovate cannot recompute the sha256 + # that must change with the installer version. + AQUA_INSTALLER_VERSION: v4.0.2 + AQUA_INSTALLER_SHA256: 98b883756cdd0a6807a8c7623404bfc3bc169275ad9064dc23a6e24ad398f43d + # renovate: depName=aquaproj/aqua + AQUA_VERSION: v2.60.1 + run: | + # Pin the root explicitly: aqua's own Windows build defaults it to + # %LOCALAPPDATA% (adrg/xdg), while the shell installer and the + # hermetic Justfile PATH compute unix-style ~/.local/share — without + # this, the layers disagree on Windows and nothing resolves. Exported + # to GITHUB_ENV so the recipes' aqua agrees too. + AQUA_ROOT_DIR="${AQUA_ROOT_DIR:-$HOME/.local/share/aquaproj-aqua}" + # On windows the pin must be in NATIVE form: git-bash's $HOME is a + # POSIX-only path (/c/Users/...) that native binaries (aqua, + # aqua-proxy, just) misread as current-drive-relative — aqua would + # link tools under D:\c\... while bash's PATH looks in C:\Users\.... + # cygpath -m yields C:/Users/..., which every layer reads correctly + # (git-bash included). + if command -v cygpath >/dev/null 2>&1; then + AQUA_ROOT_DIR="$(cygpath -m "$AQUA_ROOT_DIR")" + fi + export AQUA_ROOT_DIR + echo "AQUA_ROOT_DIR=${AQUA_ROOT_DIR}" >>"$GITHUB_ENV" + tmp="$(mktemp -d)" + curl --proto '=https' --tlsv1.2 -fsSL -o "${tmp}/aqua-installer" \ + "https://raw.githubusercontent.com/aquaproj/aqua-installer/${AQUA_INSTALLER_VERSION}/aqua-installer" + # No single digest tool exists everywhere: linux and windows git-bash + # ship coreutils sha256sum, macOS ships perl shasum. (Nothing + # aqua-managed can help here — this checksum guards the aqua + # installer itself.) + if command -v sha256sum >/dev/null 2>&1; then + echo "${AQUA_INSTALLER_SHA256} ${tmp}/aqua-installer" | sha256sum -c - + else + echo "${AQUA_INSTALLER_SHA256} ${tmp}/aqua-installer" | shasum -a 256 -c - + fi + chmod +x "${tmp}/aqua-installer" + "${tmp}/aqua-installer" -v "${AQUA_VERSION}" + rm -rf "${tmp}" + echo "${AQUA_ROOT_DIR}/bin" >>"$GITHUB_PATH" + - name: Authorize the committed aqua policy + # Before aqua will read the repo's local registry, the committed policy + # must be allowed — every caller needs this, whatever it runs next. + shell: bash + run: aqua policy allow aqua-policy.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..2067ef7 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,112 @@ +# DO NOT EDIT MANUALLY. +# This workflow is generic — no project-specific content — and is destined to +# become part of the canonical baseline limen distributes. +# +# Design: minimal GitHub glue around the same tooling every developer runs +# locally. The only marketplace action is GitHub's own checkout, pinned by +# commit SHA (a tag can be moved to malicious code; a SHA cannot). Everything +# else is pinned, checksum-verified shell: aqua installs the repo's tools at +# the versions aqua.yaml pins, and `just` runs the exact recipes a laptop +# runs — CI green means the same thing as local green, by construction. +# Deliberately absent: runner egress filtering — the available options are +# third-party actions, which this workflow avoids on principle. +name: ci + +on: + push: + branches: [main] + pull_request: + +# No default token permissions: each job states what it needs. This workflow +# only ever reads the repository — it cannot write code, releases, or +# packages even if a step is compromised. +permissions: {} + +# A superseded run (new push to the same branch/PR) is cancelled, not raced. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# Every run step is bash on every runner: without this, windows defaults to +# PowerShell, which never sees the git-bash environment setup-aqua prepares +# (the whole matrix is designed to run the recipes under git-bash). Explicit +# bash also means `-eo pipefail` everywhere, which the implicit linux/macos +# default lacks. +defaults: + run: + shell: bash + +jobs: + verify: + strategy: + fail-fast: false + matrix: + # Pinned images, not -latest: a runner bump is a reviewed change. + # macOS is not redundant: its /bin/bash is 3.2, the portability floor + # the shared recipes target. arm64 (ubuntu-24.04-arm, windows-11-arm) + # and windows cover the rest of the supported release matrix — both + # windows legs run the recipes under git-bash (native arm64 tools + # where upstream ships them, Prism-emulated amd64 via aqua's + # windows_arm_emulation where it does not). + # These names are NOT the ruleset's required contexts — the `gate` job + # below is, precisely so this list can change without touching any + # repository's branch protection. + os: [ubuntu-24.04, ubuntu-24.04-arm, macos-15, windows-2025, windows-11-arm] + runs-on: ${{ matrix.os }} + # Generous for the windows legs: those runners are markedly slower, and the + # per-GOOS lint recipes now run three legs each. + timeout-minutes: 45 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The token is not left behind in .git/config: nothing in this + # workflow talks to GitHub after checkout. + persist-credentials: false + # Full history and refs: `just do lint commits` validates the commit + # range against the PR's base branch, which a shallow clone lacks. + fetch-depth: 0 + + - name: Install aqua (pinned, checksum-verified) + uses: ./.github/actions/setup-aqua + + - name: Install pinned tools + # Link-only: the shims download each tool lazily on first use (and it + # is verified against the committed aqua-checksums.json then), so a + # job only ever pays for the tools its recipes actually run. + run: aqua install --only-link + + - name: Lint + run: just lint + + - name: Test + run: just test + + # The ONE required status check (see defaultRequiredChecks in + # internal/github/audit.go). Branch protection names contexts as strings, so + # requiring the matrix legs directly would bake this workflow's runner list + # into every repository's ruleset — and a project whose matrix differs then + # waits forever on checks that never report. This job collapses the matrix, + # whatever its shape, to a single stable name: change the legs above freely, + # the ruleset never moves. + gate: + needs: [verify] + # always(), and the result asserted explicitly. Without always() a failed + # or cancelled dependency SKIPS this job instead of failing it, and a + # skipped required check does not block a merge — branch protection that + # quietly stops protecting. `needs.verify.result` is success only when + # every matrix leg succeeded. + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: {} + steps: + - name: Every verify leg succeeded + env: + # Via env, never interpolated into the script: the shell sees data, + # not something the expression layer can rewrite into code. + RESULT: ${{ needs.verify.result }} + run: | + printf 'verify: %s\n' "$RESULT" + [ "$RESULT" = "success" ] diff --git a/.github/workflows/update-aqua-checksum.yaml b/.github/workflows/update-aqua-checksum.yaml new file mode 100644 index 0000000..9a3eb0e --- /dev/null +++ b/.github/workflows/update-aqua-checksum.yaml @@ -0,0 +1,137 @@ +# DO NOT EDIT MANUALLY. +# This workflow is generic — no project-specific content — and is destined to +# become part of the canonical baseline limen distributes. +# +# The other half of Renovate: the bot bumps versions in aqua.yaml but cannot +# do the repo-specific follow-up, so without this workflow every version-bump +# PR would merge half-applied. Two follow-ups, on pushes to Renovate's +# branches, one fix-up commit: +# 1. Regenerate aqua-checksums.json with real aqua — a bumped pin with a +# stale checksum breaks every install. +# 2. Converge the limen baseline: when the branch bumps the farcloser/limen +# pin, the repo's canonical files must move with it — a repo is coherent +# only when the limen that wrote its files is the limen it pins (an old +# baseline checked by the new limen is red, and vice versa). The +# branch's own pinned limen runs `fix`; on branches bumping anything +# else it is a no-op. +# +# This is a WRITE workflow — the hardening is deliberate: +# - `push:` on the branch prefix, never pull_request_target: it runs in the +# repo's own context, on branches only writers (the Renovate app) can +# create. +# - The checkout keeps no credential, and the update steps run with no +# secrets in their environment. Neither goes through `just` — the one +# sanctioned deviation: a write-capable workflow does not execute recipe +# code the branch controls. `aqua update-checksum` only downloads and +# hashes declared artifacts; the converge step executes exactly one +# binary, the checksum-pinned limen release the branch declares. +# - The push step runs only git, with the token scoped to that single step. +# - The branch name reaches the shell via env, never template interpolation +# (script-injection hygiene). +# - No loop: a push made with the default GITHUB_TOKEN triggers no further +# workflows — and the no-change early exit terminates recursion +# regardless. +# +# Known trade of the default token: GitHub suppresses workflow runs for +# commits it pushes, so the PR's CI does not re-run on the checksum commit. +# To get CI on the final state of Renovate PRs, register a GitHub App — +# contents:write only, webhook disabled, installed on the org; no infra, it +# is just an identity — and set the org variable UPDATE_AQUA_CHECKSUM_APP_ID +# plus the org secret UPDATE_AQUA_CHECKSUM_APP_PRIVATE_KEY: the workflow +# then mints a one-hour, this-repo-only token per run, so there is no +# long-lived broad credential and nothing that expires on a calendar. A +# fine-grained PAT with contents:write as UPDATE_AQUA_CHECKSUM_TOKEN is the +# drop-in alternative; the token preference order is App, PAT, default. +name: update-aqua-checksum + +on: + push: + branches: ["renovate/**"] + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# Every run step is bash, explicitly: runner-OS shell defaults are a trap +# (windows defaults to PowerShell), and explicit bash adds `-eo pipefail`. +# Uniform across all canonical workflows so adding a runner never changes +# what the steps mean. +defaults: + run: + shell: bash + +jobs: + update: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install aqua (pinned, checksum-verified) + uses: ./.github/actions/setup-aqua + + - name: Regenerate aqua-checksums.json + # No tool install: update-checksum only reads the manifest and hashes + # upstream artifacts — the smallest possible surface for a write job. + # Runs first: it is what makes the branch's limen pin installable for + # the converge step below. + run: aqua update-checksum --prune + + - name: Converge the limen baseline + # `aqua exec` resolves the branch's own farcloser/limen pin (checksums + # fresh from the step above), downloads it verified, and `limen fix` + # rewrites whatever the new baseline moved. A released limen leaves + # its own pin alone (it already matches), so this cannot ping-pong. + # Skipped in the limen repository itself: there the working tree IS + # the next baseline (the recipes run it via LIMEN_BIN), and a released + # limen "converging" it would revert in-flight baseline work. + if: github.repository != 'farcloser/limen' + run: aqua exec -- limen fix . + + - name: Mint a push token, if the App is configured + # Placed after the update steps so those still run with no secrets in + # their environment. The minted token lives one hour and is scoped to + # this repository only; the long-lived private key exists solely to + # mint and never authorizes a push itself. Guarded on the variable so + # repos without the App fall through to the token chain below. + if: vars.UPDATE_AQUA_CHECKSUM_APP_ID != '' + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ vars.UPDATE_AQUA_CHECKSUM_APP_ID }} + private-key: ${{ secrets.UPDATE_AQUA_CHECKSUM_APP_PRIVATE_KEY }} + # Least privilege: the minted token only ever pushes a commit, so scope it + # to contents:write rather than inheriting whatever the App installation + # happens to hold now or later. + permission-contents: write + + - name: Push the update, if any + env: + BRANCH: ${{ github.ref_name }} + TOKEN: ${{ steps.app-token.outputs.token || secrets.UPDATE_AQUA_CHECKSUM_TOKEN || github.token }} + run: | + [ -z "$(git status --porcelain)" ] && { echo "checksums and baseline already in sync"; exit 0; } + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # `--all` is deliberate, not sloppiness. The tree is fully accounted + # for: a fresh checkout, aqua rooted outside the workspace, and + # exactly two writers between checkout and here — update-checksum + # and the pinned limen's fix. Whatever is dirty IS the payload. An + # enumerated path list would be wrong: the converge step's job is to + # commit whatever the NEW limen's baseline says, and a list baked + # into the older running workflow cannot know that surface (the + # updated workflow arrives in the very commit being built). Nor + # would a list add safety — the only writer that could plant a file + # is limen fix itself, and .limen/.github would be on any list. + git add --all + # Signed-off-by: `just do lint commits` enforces DCO on the PR range, + # bot commits included. + git commit -m "chore: update aqua checksums and converge the limen baseline" \ + -m "Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" + git push "https://x-access-token:${TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "HEAD:${BRANCH}" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..61ed41a --- /dev/null +++ b/.gitignore @@ -0,0 +1,53 @@ +# --- OS noise --- +.DS_Store +Thumbs.db +Desktop.ini + +# --- Editor / IDE --- +.idea/ +.vscode/ +*.swp +*~ + +# Build output and Go artifacts +/build +/tmp + +# Scratch / work-in-progress files +# *.local +# _scratch/ +# WIP_* + +# ========================= +# Rust +# ========================= +# Cargo build output +# /target/ + +# Cargo.lock: COMMIT for binaries/apps, IGNORE for libraries. +# Uncomment the next line ONLY if this is a published library crate: +# Cargo.lock +# Backup files from `cargo fmt` / rustfmt edits +# **/*.rs.bk + +# MSVC debug info (Windows toolchain) +# *.pdb + +# ========================= +# Svelte / SvelteKit / Node +# ========================= +# Dependencies +# node_modules/ + +# SvelteKit build + generated types/runtime +# .svelte-kit/ + +# Vite cache/output +# .vite/ +# vite.config.*.timestamp-* + +# Package manager debug logs +# npm-debug.log* +# yarn-debug.log* +# yarn-error.log* +# pnpm-debug.log* diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..4ed5b7b --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,117 @@ +version: "2" + +# Linters for a cryptographic hash library. The bias is towards correctness +# and away from style: checks that would fight the shape of the code (tight +# compression loops, deliberate width conversions, unsafe casts feeding SIMD +# kernels) are off, and checks that catch real logic and error-handling +# mistakes are on. + +linters: + enable: + # Error handling. + - errorlint # == and type switches on errors break once anything wraps + - nilerr # returning nil after checking an error is never intended + + # Correctness traps that are easy to introduce and hard to see. + - bodyclose + - copyloopvar + - durationcheck + - makezero + - predeclared # shadowing min/max/clear is a live hazard since Go 1.21 + - reassign + - wastedassign + + # General analysis. + - gocritic + - gosec + - revive + + # This package tracks its own performance closely (zero-allocation hot + # paths); fmt.Sprintf in one of them would be a real regression. + - perfsprint + + # Hygiene. + - misspell + - nolintlint + - unconvert + - usestdlibvars + + # Tests. + - thelper + - tparallel + + settings: + errcheck: + exclude-functions: + # Infallible by implementation: Hasher.Write always returns nil (the + # error exists to satisfy hash.Hash), and OutputReader.Read can only + # fail at the end of its 2^64-byte stream. + - (*github.com/forkcloser/blake3.Hasher).Write + - (*github.com/forkcloser/blake3.OutputReader).Read + + gosec: + excludes: + # The hash state is words; the wire format is bytes; stream offsets + # are uint64 sliced by int lengths. Width conversions are what this + # package does — the ones that need bounds guards have them, and are + # covered by tests down to the overflow edges (see TestXOFSeek). + - G115 + # guts converts between [64]byte and [16]uint32 views of the same + # memory with unsafe on amd64 — that is the SIMD interface, it is + # deliberate, and it was reviewed as part of the audit. Flagging + # every cast buries the signal. + - G103 + # Duplicates errcheck, which reports the same thing more precisely. + - G104 + + nolintlint: + require-explanation: true + require-specific: true + + staticcheck: + checks: + - all + + revive: + rules: + - name: redefines-builtin-id + - name: blank-imports + - name: context-as-argument + - name: context-keys-type + - name: dot-imports + - name: empty-block + - name: error-naming + - name: error-return + - name: error-strings + - name: errorf + - name: increment-decrement + - name: indent-error-flow + - name: range + - name: receiver-naming + - name: superfluous-else + - name: time-naming + - name: unexported-return + - name: unreachable-code + - name: var-declaration + - name: waitgroup-by-value + + exclusions: + rules: + # Tests want a seeded, reproducible PRNG (split-point fuzzing, read + # patterns) — the outputs are compared against golden values, so + # cryptographic randomness would only make failures unreproducible. + - path: _test\.go + linters: [gosec] + text: G404 + + # hash.Hash.Write never returns an error by contract, and the XOF's + # Read/Seek only fail at the end of a 2^64-byte stream or on invalid + # offsets the tests construct deliberately. The tests exercise these + # APIs constantly; checking every call would bury the assertions that + # matter, and a real failure still surfaces as a wrong digest. + - path: _test\.go + linters: [errcheck] + +formatters: + enable: + - gofmt diff --git a/.limen/.shellcheckrc b/.limen/.shellcheckrc new file mode 100644 index 0000000..4bc2aa1 --- /dev/null +++ b/.limen/.shellcheckrc @@ -0,0 +1,17 @@ +# DO NOT EDIT MANUALLY. +# This file is common to all projects and managed by limen. +# If you need local overrides for this project, you can use inline `# shellcheck disable=SCXXXX`. +# Global configuration changes proposals can be discussed on https://github.com/farcloser/limen + +# .shellcheckrc — https://www.shellcheck.net/wiki/ + +# Follow `source`/`.`-ed files so shellcheck checks across includes. +# 'true' lets it follow even non-constant source paths (best for repos with libs). +external-sources=true + +# Opt INTO the optional checks shellcheck ships but doesn't run by default. +enable=quote-safe-variables # flags unquoted vars that should be quoted +# enable=require-variable-braces # enforce ${var} consistently (stylistic — deliberately off) +enable=check-unassigned-uppercase +enable=deprecate-which # `which` → `command -v` +enable=avoid-nullary-conditions diff --git a/.limen/.yamlfmt b/.limen/.yamlfmt new file mode 100644 index 0000000..8269a9c --- /dev/null +++ b/.limen/.yamlfmt @@ -0,0 +1,56 @@ +# DO NOT EDIT MANUALLY. +# This file is common to all projects and managed by limen. +# Global configuration changes proposals can be discussed on https://github.com/farcloser/limen + +# .yamlfmt — https://github.com/google/yamlfmt +# Formatter: the 'basic' formatter is the only one; these are its options. +formatter: + type: basic + + # LF everywhere, explicitly: yamlfmt's default flips to CRLF on Windows, + # which would flag every (correctly LF) file. Same doctrine as the pinned + # .editorconfig and .gitattributes: line endings never vary by platform. + line_ending: lf + + # --- indentation --- + indent: 2 # match your .editorconfig (2-space YAML) + include_document_start: false # don't force a leading `---` on every file + + # --- the defaults worth overriding --- + retain_line_breaks_single: true # collapse runs of blank lines to ONE, but keep + # intentional single blank lines (readability). + # Prefer this over retain_line_breaks, which keeps + # *all* blank lines and barely normalizes anything. + + scan_folded_as_literal: false # leave folded (>) scalars folded; don't rewrite to literal (|) + + # --- correctness / safety --- + disallow_anchors: false # set true ONLY if you want to forbid &anchors/*aliases + max_line_length: 0 # 0 = no wrapping. Leave OFF — yamlfmt's wrapping is + # crude and mangles long values; let humans wrap. + + # --- quoting / strings: leave alone --- + # yamlfmt is deliberately light on string normalization. Don't fight it; it won't + # aggressively re-quote, which is the safe default for mixed YAML (k8s, CI, etc.) + +# --- which files to format --- +# Every pattern below is doublestar syntax (`**/`, `{yaml,yml}` braces), which +# yamlfmt does NOT use unless told to: without this switch it hands patterns to +# the filesystem, where POSIX shrugs (ENOENT, silently no-op) but Windows +# hard-fails (`CreateFile .git/**: ... syntax is incorrect`). +doublestar: true + +include: + - "**/*.{yaml,yml}" + +gitignore_excludes: true + +exclude: + - ".git/**" + - "**/vendor/**" + - "**/testdata/**" # don't reformat fixtures — tests may assert exact bytes + - "**/*.gen.{yaml,yml}" # leave generated YAML alone + +# Stop at a file that fails to parse — malformed YAML must fail hard, in CI +# and everywhere else. +continue_on_error: false diff --git a/.limen/aqua-registry.yaml b/.limen/aqua-registry.yaml new file mode 100644 index 0000000..b80c4ce --- /dev/null +++ b/.limen/aqua-registry.yaml @@ -0,0 +1,94 @@ +# DO NOT EDIT MANUALLY. +# This file is common to all projects and managed by limen. +# Most tools can be installed directly from aqua registry without local overrides. +# Addition of non-standard tools can be discussed on https://github.com/farcloser/limen + +packages: + # The /v2 module-path suffix is mandatory (Go semantic import versioning): + # `go install github.com/google/go-licenses@v2.x` is rejected by the + # toolchain. Go strips the /v2 when naming the binary; files spells it out. + - type: go_install + path: github.com/google/go-licenses/v2 + description: Report on the licenses of a Go project's dependencies + version_source: github_tag + files: + - name: go-licenses + - type: go_install + path: github.com/vbatts/git-validation + description: Validate git commit rules (DCO sign-off, subject length, …) + version_source: github_tag + # Graphviz `dot` as a single static Go binary: the real graphviz C code + # compiled to WASM, executed via wazero — no C toolchain, no shared + # libraries, which is why this can live in aqua while system graphviz + # cannot. Renders the PNGs of `just do test go profile`. The cmd/dot module is + # nested and untagged upstream, hence the explicit repo and the + # pseudo-version pin in aqua.yaml. + - type: go_install + name: github.com/goccy/go-graphviz/cmd/dot + path: github.com/goccy/go-graphviz/cmd/dot + repo_owner: goccy + repo_name: go-graphviz + description: Graphviz dot CLI, pure-Go build (WASM via wazero) + version_source: github_tag + # The Go team distributes govulncheck via `go install` only, by policy. The + # repo is not inferable from the golang.org import path, hence the explicit + # owner/name (same as the standard registry does for gopls). + - type: go_install + name: golang.org/x/vuln/cmd/govulncheck + path: golang.org/x/vuln/cmd/govulncheck + repo_owner: golang + repo_name: vuln + description: Scan dependencies against the Go vulnerability database + version_source: github_tag + # Same golang.org path situation as govulncheck: explicit name and repo. + - type: go_install + name: golang.org/x/tools/cmd/deadcode + path: golang.org/x/tools/cmd/deadcode + repo_owner: golang + repo_name: tools + description: Whole-program detection of unreachable functions + version_source: github_tag + # hadolint as pure Go, from the farcloser stable. The binary is the nested + # cmd/godolint package of the root module — explicit name and repo, same + # shape as go-graphviz's cmd/dot above (tags live at the repo root). + - type: go_install + name: github.com/farcloser/godolint/cmd/godolint + path: github.com/farcloser/godolint/cmd/godolint + repo_owner: farcloser + repo_name: godolint + description: Lint Dockerfiles (a pure-Go port of hadolint) + version_source: github_tag + # limen itself: every repo pins the version that enforces it, so the binary + # and the canonical files it embeds travel together (no version skew between + # the checker and the checked). Graduates to the standard registry once + # farcloser/limen is registered there. + - type: github_release + repo_owner: farcloser + repo_name: limen + description: Verify a repository against Farcloser engineering rules + asset: limen_{{trimV .Version}}_{{.OS}}_{{.Arch}}.tar.gz + format: tar.gz + files: + - name: limen + checksum: + type: github_release + asset: checksums.txt + algorithm: sha256 + # Signatures are verified whenever upstream publishes them (see the + # book's signature doctrine): the release signs checksums.txt keyless + # from CI (goreleaser cosign lane), which transitively covers every + # asset the file lists. The identity IS the release workflow at the + # exact tag — a bundle signed by anything else fails the install. + cosign: + bundle: + type: github_release + asset: checksums.txt.sigstore.json + opts: + - --certificate-identity + - https://github.com/farcloser/limen/.github/workflows/release.yaml@refs/tags/{{.Version}} + - --certificate-oidc-issuer + - https://token.actions.githubusercontent.com + supported_envs: + - darwin/arm64 + - linux + - windows diff --git a/.limen/just/build-go.just b/.limen/just/build-go.just new file mode 100644 index 0000000..60977ee --- /dev/null +++ b/.limen/just/build-go.just @@ -0,0 +1,176 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +# Ported from the historical Make build system, corrected along the way: +# - a "binary" is a DIRECTORY under cmd/ that contains Go sources — a stray +# file or docs directory no longer becomes a phantom target; +# - version stamping is reproducible: main.version carries `git describe`, +# and commit/date come from Go's own VCS embedding (`go version -m`) at +# their COMMIT values — never wall-clock build time, which made the same +# commit produce different binaries; +# - build tags are scoped to these commands, never exported globally where +# they would leak into tests and linters; +# - static states its preconditions (Linux, CGO) and fails fast instead of +# dying with a cryptic linker error; +# - a race build exists. +# Binaries land in build/ (covered by the canonical .gitignore). Projects +# declare `var version = "dev"` in each main package for the stamp to land. +# Extra flags: export BUILD_GO_FLAGS from the root Justfile, the home of all +# project customization (the variable name mirrors the task path), or set it +# on the invocation for a one-off. +# +# Extra LINKER flags: BUILD_GO_LDFLAGS, same convention. It exists because +# -ldflags cannot come through BUILD_GO_FLAGS: go takes the LAST occurrence of a +# repeated flag, and BUILD_GO_FLAGS is appended after the -ldflags these recipes +# compute, so a project passing its own would silently REPLACE the version stamp +# and the CGO linkmode rather than add to them. BUILD_GO_LDFLAGS is appended +# INSIDE the -ldflags string instead, last, so it composes — and, being last, +# also wins on a genuine conflict (a project overriding -X main.version). +# Typical use is a second stamp the project needs pinned at link time: +# export BUILD_GO_LDFLAGS := '-X main.someImage=' + some_digest_pinned_ref + +# netgo/osusergo force the pure-Go resolver and user lookups. No-ops when +# CGO_ENABLED=0 (pure Go already); with CGO they keep DNS and user handling +# out of libc, which is exactly what makes `static` viable on glibc. +go_tags := 'netgo,osusergo' + +# CGO is off unless asked for: per invocation with `CGO_ENABLED=1 just do build +# go`, or for good with `export GO_CGO := '1'` in the root Justfile, for a project +# whose product cannot link without it. Every recipe below resolves it through +# lib.just's `go_cgo`, which documents the precedence and why it is a shell +# expansion rather than a just variable. + +# --- C toolchain hardening — only effective when CGO_ENABLED=1 -------------- +# The canonical hardening set (see the Red Hat compiler-flags guidance), +# platform-gated: stack-clash protection and the -z linker set are Linux-only +# (ld64 on macOS supports neither). +c_warnings := '-Wall -Werror=format-security' +c_security := '-fstack-protector-strong -fPIE -D_FORTIFY_SOURCE=2' + (if os() == 'linux' { ' -fstack-clash-protection' } else { '' }) +cgo_cflags_release := c_warnings + ' -O2 ' + c_security + ' -pipe' + +# Debug: -O0 with real debug info. FORTIFY_SOURCE is deliberately absent — it +# requires -O1 or higher and would only produce warnings at -O0. +cgo_cflags_debug := c_warnings + ' -O0 -g -grecord-gcc-switches -pipe' + +# libstdc++ assertions are a C++-only concern: valid in CXXFLAGS, not CFLAGS. +cgo_cxxflags_debug := cgo_cflags_debug + ' -D_GLIBCXX_ASSERTIONS' +cgo_ldflags := if os() == 'linux' { '-Wl,-z,defs -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack' } else { '' } + +default: release + +# Release: reproducible (trimpath, commit-time VCS stamp), stripped, PIE. +# Twin of the builds section in .goreleaser.yaml (the release builder, where +# a project ships one) — a flag change here must land there too. See the +# comment there for the differences that are intended. +release: (_banner "build go" "release") + #!/usr/bin/env bash + set -euo pipefail + export CGO_ENABLED="{{ go_cgo }}" + version=$(git describe --tags --always --dirty 2>/dev/null || echo dev) + ldflags="-s -w -X main.version=${version}" + if [ "${CGO_ENABLED:-0}" = "1" ]; then + export CGO_CFLAGS='{{ cgo_cflags_release }}' + export CGO_CXXFLAGS='{{ cgo_cflags_release }}' + export CGO_LDFLAGS='{{ cgo_ldflags }}' + ldflags="-linkmode=external -extldflags=-pie ${ldflags}" + fi + ldflags="${ldflags} ${BUILD_GO_LDFLAGS:-}" + extra=() + [ -z "${BUILD_GO_FLAGS:-}" ] || read -ra extra <<<"${BUILD_GO_FLAGS}" + mkdir -p build + for dir in cmd/*/; do + name=$(basename "${dir}") + ls "${dir}"*.go >/dev/null 2>&1 || continue + echo "→ build/${name}" + go build -trimpath -buildmode=pie -tags='{{ go_tags }}' \ + -ldflags "${ldflags}" ${extra[@]+"${extra[@]}"} -o "build/${name}" "./${dir%/}" + done + +# Debug: optimizations and inlining off, symbols kept, real paths kept — no +# trimpath, deliberately: debuggers want actual file locations. -debug suffix. +debug: (_banner "build go" "debug") + #!/usr/bin/env bash + set -euo pipefail + export CGO_ENABLED="{{ go_cgo }}" + version=$(git describe --tags --always --dirty 2>/dev/null || echo dev) + ldflags="-X main.version=${version}" + if [ "${CGO_ENABLED:-0}" = "1" ]; then + export CGO_CFLAGS='{{ cgo_cflags_debug }}' + export CGO_CXXFLAGS='{{ cgo_cxxflags_debug }}' + export CGO_LDFLAGS='{{ cgo_ldflags }}' + ldflags="-linkmode=external -extldflags=-pie ${ldflags}" + fi + ldflags="${ldflags} ${BUILD_GO_LDFLAGS:-}" + extra=() + [ -z "${BUILD_GO_FLAGS:-}" ] || read -ra extra <<<"${BUILD_GO_FLAGS}" + mkdir -p build + for dir in cmd/*/; do + name=$(basename "${dir}") + ls "${dir}"*.go >/dev/null 2>&1 || continue + echo "→ build/${name}-debug" + go build -buildmode=pie -gcflags='all=-N -l' -tags='{{ go_tags }}' \ + -ldflags "${ldflags}" ${extra[@]+"${extra[@]}"} -o "build/${name}-debug" "./${dir%/}" + done + +# Race: a diagnostic build with the race detector, which requires cgo (forced +# here) and, matching the test module, an external linkmode. Unstripped, real +# paths. -race suffix. +race: (_banner "build go" "race") + #!/usr/bin/env bash + set -euo pipefail + export CGO_ENABLED=1 + version=$(git describe --tags --always --dirty 2>/dev/null || echo dev) + ldflags="-linkmode=external -X main.version=${version} ${BUILD_GO_LDFLAGS:-}" + extra=() + [ -z "${BUILD_GO_FLAGS:-}" ] || read -ra extra <<<"${BUILD_GO_FLAGS}" + mkdir -p build + for dir in cmd/*/; do + name=$(basename "${dir}") + ls "${dir}"*.go >/dev/null 2>&1 || continue + echo "→ build/${name}-race" + go build -race -tags='{{ go_tags }}' \ + -ldflags "${ldflags}" \ + ${extra[@]+"${extra[@]}"} -o "build/${name}-race" "./${dir%/}" + done + +# Static: fully static external link. Preconditions enforced, not commented: +# Linux only (macOS has no static libc), and CGO_ENABLED=1 (with CGO off, +# every build is already static — use `release`). No PIE: classic static and +# PIE conflict. -static suffix. +static: (_banner "build go" "static") + #!/usr/bin/env bash + set -euo pipefail + export CGO_ENABLED="{{ go_cgo }}" + if [ "$(uname -s)" != "Linux" ]; then + echo "static builds require Linux: macOS has no static libc to link against." >&2 + exit 1 + fi + if [ "${CGO_ENABLED:-0}" != "1" ]; then + echo "static needs CGO_ENABLED=1 — a pure-Go build is already static (use \`just do build go\`)." >&2 + exit 1 + fi + version=$(git describe --tags --always --dirty 2>/dev/null || echo dev) + export CGO_CFLAGS='{{ cgo_cflags_release }}' + export CGO_CXXFLAGS='{{ cgo_cflags_release }}' + export CGO_LDFLAGS='{{ cgo_ldflags }}' + ldflags="-linkmode=external -extldflags=-static -s -w -X main.version=${version} ${BUILD_GO_LDFLAGS:-}" + extra=() + [ -z "${BUILD_GO_FLAGS:-}" ] || read -ra extra <<<"${BUILD_GO_FLAGS}" + mkdir -p build + for dir in cmd/*/; do + name=$(basename "${dir}") + ls "${dir}"*.go >/dev/null 2>&1 || continue + echo "→ build/${name}-static" + go build -trimpath -tags='{{ go_tags }}' \ + -ldflags "${ldflags}" \ + ${extra[@]+"${extra[@]}"} -o "build/${name}-static" "./${dir%/}" + done diff --git a/.limen/just/build.just b/.limen/just/build.just new file mode 100644 index 0000000..9582374 --- /dev/null +++ b/.limen/just/build.just @@ -0,0 +1,16 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Go builds live in their own submodule: `just do build go` makes the release +# binaries, `just do build go ` (debug, race, static) makes a variant. +mod go 'build-go.just' + +# A submodule's recipe IS addressable as a dependency (go::default); only a +# bare module path (go) is not. +default: go::default diff --git a/.limen/just/do.just b/.limen/just/do.just new file mode 100644 index 0000000..34a69d2 --- /dev/null +++ b/.limen/just/do.just @@ -0,0 +1,25 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# The `do` namespace holds every shared task, so the top level stays free for +# a project's own recipes (a project may define its own `just lint`; the +# shared one is `just do lint`). + +# Show the shared tasks. +default: + @just --list do + +mod build 'build.just' +mod tools 'tools.just' +mod lint 'lint.just' +mod test 'test.just' +mod fix 'fix.just' + +# Flat (imported, not a module) so it takes arguments: `just do release v1.2.3`. +import 'release.just' diff --git a/.limen/just/fix-go.just b/.limen/just/fix-go.just new file mode 100644 index 0000000..6662c3e --- /dev/null +++ b/.limen/just/fix-go.just @@ -0,0 +1,54 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +default: code mod + +# golangci-lint auto-fixes, once per supported platform (see _per-goos in +# lib.just: platform-specific files are only analyzed for the GOOS that +# builds them). The formatter is build-graph-independent and runs last, so the +# fixes' edits end up formatted — but it runs to CONVERGENCE, not once. +# +# `golangci-lint fmt` applies every enabled formatter in a single pass, and the +# set is not confluent: one formatter's output can be work for another (gofumpt +# wanting a blank line that golines' rewrite exposed, say). One pass can +# therefore leave a tree that `fmt --diff` still rejects — so `just fix` would +# report success and `just lint` immediately fail, which is the worst possible +# pairing. Observed on protoc-gen-go output, which needs exactly two passes. +# +# Bounded, because non-convergence has two causes and only one is benign: a +# formatter set that merely needs another pass settles in two or three, while +# two formatters that undo each other never settle. Loop for the first, fail +# loudly for the second instead of hanging CI forever. +code: (_banner "fix go" "go") (_per-goos "golangci-lint run --fix") + #!/usr/bin/env bash + set -euo pipefail + max=5 + pass=0 + # Check first: an already-formatted tree costs one --diff and no rewrite. + while ! golangci-lint fmt --diff >/dev/null 2>&1; do + pass=$((pass + 1)) + if [ "${pass}" -gt "${max}" ]; then + echo "formatters did not converge after ${max} passes — they are fighting each other." >&2 + echo "The residual diff below reverses on the next pass; fix the formatter config." >&2 + golangci-lint fmt --diff >&2 || true + exit 1 + fi + golangci-lint fmt + done + +# Tidy go.mod/go.sum. +mod: (_banner "fix go" "mod") + go mod tidy + +# Update every dependency to its latest version, then tidy. +up: (_banner "fix go" "up") && mod + go get -u ./... diff --git a/.limen/just/fix-homebrew.just b/.limen/just/fix-homebrew.just new file mode 100644 index 0000000..c154dbe --- /dev/null +++ b/.limen/just/fix-homebrew.just @@ -0,0 +1,67 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +# The mutating counterpart of `just do lint homebrew` — brew's vendored +# RuboCop with --fix. Same machine-layer exception as the lint module: +# BREW_BIN is captured from the ambient PATH at just startup, never from the +# hermetic PATH (the reasoning lives in lint-homebrew.just and main.just). + +default: style + +# Repair what `just do lint homebrew style` reports; passes vacuously when the +# repository carries no formulas or casks. +style: (_banner "fix homebrew" "style") + #!/usr/bin/env bash + set -euo pipefail + {{ _skip_unless_macos }} + # --cached --others --exclude-standard: tracked *and* new untracked files, + # still honoring .gitignore (same as `fix just`). Modern tap layout ONLY — + # Formula/ and Casks/; the legacy locations are deliberately unsupported + # (see lint-homebrew.just). Discovery runs before the brew lookup so a + # repo without formulas passes vacuously even brew-less. + files=() + while IFS= read -r -d '' f; do + # Tracked-but-deleted files are skipped (see lint.just's `just` recipe). + [ -f "$f" ] || continue + files+=("$f") + done < <(git ls-files -z --cached --others --exclude-standard \ + 'Formula/*.rb' 'Formula/**/*.rb' 'Casks/*.rb' 'Casks/**/*.rb') + if [ "${#files[@]}" -eq 0 ]; then + echo "no formulas or casks (Formula/, Casks/) — nothing to fix." + exit 0 + fi + {{ _require_brew }} + # shellcheck disable=SC2154 # BREW_BIN is exported by the canonical Justfile (main.just). + "$BREW_BIN" style --fix "${files[@]}" + +# Homebrew exists on macOS only: everywhere else this recipe degrades to a +# LOUD no-op — the twin of the variable in lint-homebrew.just; a change here +# must land there too. +[private] +_skip_unless_macos := ''' + if [ "$(uname -s)" != 'Darwin' ]; then + echo "homebrew is macOS-only — nothing to do on this platform (skipped, not failed)." + exit 0 + fi +''' + +# BREW_BIN comes from main.just (ambient-PATH capture) — the twin of the +# variable in lint-homebrew.just; a change here must land there too. +[private] +_require_brew := ''' + if [ -z "${BREW_BIN:-}" ]; then + echo "brew was not found on your PATH when just started." >&2 + echo "brew is a machine-layer tool — install it outside the repo (or export" >&2 + echo "BREW_BIN=/path/to/brew), then re-run." >&2 + exit 1 + fi +''' diff --git a/.limen/just/fix-rust.just b/.limen/just/fix-rust.just new file mode 100644 index 0000000..df77a6b --- /dev/null +++ b/.limen/just/fix-rust.just @@ -0,0 +1,15 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +rust: (_banner "fix" "rust") + cargo clippy --fix --all-targets --all-features --allow-dirty --allow-staged + cargo fmt --all diff --git a/.limen/just/fix.just b/.limen/just/fix.just new file mode 100644 index 0000000..19e1a74 --- /dev/null +++ b/.limen/just/fix.just @@ -0,0 +1,57 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +default: limen just yaml aqua + +# limen itself: repair what `just do lint limen` reports — rewrite drifted +# canonical files, create missing mandatory ones, restore aqua pins and +# regenerate aqua-checksums.json (network, when pins changed). Same LIMEN_BIN +# override as the lint recipe (see lint.just). +limen: (_banner "fix" "limen") + ${LIMEN_BIN:-limen} fix . + +# Repair what `just do lint github` reports — plans first, applies on consent +# (pass -yes for unattended use, -org for the organization). Never in +# the default set, same reasoning as the lint twin. +github *args: (_banner "fix" "github") + ${LIMEN_BIN:-limen} github fix {{ args }} + +just: (_banner "fix" "just") + #!/usr/bin/env bash + set -euo pipefail + # --cached --others --exclude-standard: tracked *and* new untracked files, + # still honoring .gitignore — so a not-yet-staged file is formatted too. + # Tracked-but-deleted files are skipped (see `lint just` for the rationale: + # enumeration is git's, truth is the working tree's). + # Same pathspecs as `lint just`: literals carry their '**/' twins (git + # pathspec literals are case-sensitive and do not match across + # directories), and the hidden .justfile spelling is covered. + while IFS= read -r -d '' f; do + [ -f "$f" ] || continue + just --fmt --justfile "$f" + done < <(git ls-files -z --cached --others --exclude-standard \ + 'justfile' '**/justfile' 'Justfile' '**/Justfile' '.justfile' '**/.justfile' '*.just') + +yaml: (_banner "fix" "yaml") + yamlfmt -conf .limen/.yamlfmt + +# Regenerate aqua-checksums.json from aqua.yaml — the mutating twin of +# `just do lint aqua` (needs network). In the default set so a plain +# `just do fix` heals the drift the lint default flags. +aqua: (_banner "fix" "aqua") + aqua --log-level warn update-checksum --prune + +# Go fixers live in their own submodule: `just do fix go` runs them all, +# `just do fix go ` (e.g. `just do fix go mod`) runs one. +mod go 'fix-go.just' +mod rust 'fix-rust.just' +mod homebrew 'fix-homebrew.just' diff --git a/.limen/just/lib.just b/.limen/just/lib.just new file mode 100644 index 0000000..15f421e --- /dev/null +++ b/.limen/just/lib.just @@ -0,0 +1,66 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +# Uniform per-recipe banner, shared across the lint/fix/tools modules via `import` +# (a `mod` can't be depended on, and modules don't inherit the parent's recipes). +# Private → hidden from --list. The module name is passed in because just has no +# current-module variable; depended on with arguments so it prints identically for +# plain *and* shebang recipes, which just otherwise echoes inconsistently (plain +# recipes echo their lines; shebang recipes do not). +_banner mod name: + @echo "▶ {{ mod }}: {{ name }}" + +# go_cgo answers "does THIS project build with cgo?" for every Go recipe — +# `export CGO_ENABLED="{{ go_cgo }}"` in a shebang body, or as a command prefix in +# a plain one. Interpolated into build, test and the per-platform analysis legs so +# they cannot disagree about the nature of the project. +# +# It is a SHELL EXPANSION, not a just value, and that is the entire design: it is +# evaluated when the recipe runs, where a parent's exports are present. Resolving +# it at parse time instead — `export CGO_ENABLED := env('CGO_ENABLED', '0')`, how +# this once read — cannot work: a child module cannot see its parent's variables, +# and env() reads just's PROCESS environment, which never holds one, so a project's +# `export GO_CGO := '1'` was silently discarded and 0 re-exported over it. +# +# Precedence, in order: an explicit CGO_ENABLED on the invocation wins (including +# downgrading a cgo project for a one-off pure-Go run), then the project's GO_CGO, +# then off — because go's own default is cgo ON for native builds wherever a C +# toolchain happens to exist, which would make the same tree behave differently on +# two machines. +go_cgo := '${CGO_ENABLED:-${GO_CGO:-0}}' + +# Run CMD once per supported platform (the list below mirrors the canonical +# checksum.supported_envs in aqua.yaml) with CGO disabled: the Go build/dependency graph +# differs per GOOS, so a single native run misses the other platforms' files +# and dependencies. CGO prevents exactly that — loading packages for a foreign +# GOOS needs that platform's C toolchain — so a project that genuinely needs +# cgo declares it (`export GO_CGO := '1'`, or CGO_ENABLED=1 for one invocation) +# and gets one native run only: reduced coverage, announced loudly. +# The same resolution as the build module, so ONE project knob governs every Go +# task — analysis and build agree on whether this project is a cgo project. +_per-goos +cmd: + #!/usr/bin/env bash + set -euo pipefail + if [ "{{ go_cgo }}" = "1" ]; then + echo "CGO_ENABLED=1: cross-platform runs are impossible (foreign-GOOS package" >&2 + echo "loading needs that platform's C toolchain) — native platform only." >&2 + {{ cmd }} + else + # Native platform FIRST, always: go_install tools build lazily on + # first use, and `go install` refuses to cross-compile when GOBIN is + # set (which is how aqua builds Go tools) — so a cold start must + # trigger that build on the native leg; the foreign legs then merely + # run the already-built binary. + # Extend this list together with checksum.supported_envs in aqua.yaml. + platforms=(darwin linux windows) + native="$(go env GOHOSTOS)" + ordered=("$native") + for goos in "${platforms[@]}"; do + [ "$goos" = "$native" ] || ordered+=("$goos") + done + for goos in "${ordered[@]}"; do + echo "→ GOOS=$goos" + CGO_ENABLED=0 GOOS="$goos" {{ cmd }} + done + fi diff --git a/.limen/just/lint-go.just b/.limen/just/lint-go.just new file mode 100644 index 0000000..fef0795 --- /dev/null +++ b/.limen/just/lint-go.just @@ -0,0 +1,115 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +default: code mod vuln licenses + +# golangci-lint, once per supported platform (see _per-goos in lib.just: +# platform-specific files are only analyzed for the GOOS that builds them). +# The formatter is build-graph-independent and runs once. The last step bans +# blanket revive suppressions: nolintlint polices golangci directives, and +# revive findings proved environment-nondeterministic across the per-GOOS +# legs — the suppression itself then flakes as "unused". revive's own +# selective directives are invisible to nolintlint, hence stable. +code: (_banner "lint go" "go") (_per-goos "golangci-lint run") + golangci-lint fmt --diff + @if git grep -nE 'nolint:[a-z, ]*revive' -- '*.go'; then echo 'blanket revive suppression is banned — use //revive:disable-next-line: (or a disable/enable block) instead' >&2; exit 1; fi + +# go.mod/go.sum tidiness — fails printing the diff; `just do fix go mod` repairs. +mod: (_banner "lint go" "mod") + go mod tidy -diff + +# Known-vulnerability scan against the Go vulnerability database (network: +# govulncheck fetches vuln.go.dev), once per supported platform (see _per-goos +# in lib.just: call paths and dependencies differ by GOOS). +vuln: (_banner "lint go" "vuln") (_per-goos "govulncheck ./...") + +# Dependency license compliance, once per supported platform (see _per-goos in +# lib.just: each GOOS pulls its own dependencies). A project that needs extra +# flags — typically --ignore= for the module-layout false positives of +# https://github.com/google/go-licenses/issues/186 — exports them from its +# root Justfile, the home of all project customization (the variable name +# mirrors the task path `lint go licenses`): +# export LINT_GO_LICENSES_FLAGS := '--ignore=gotest.tools/v3' +# For a one-off, set it on the invocation instead: +# LINT_GO_LICENSES_FLAGS='--ignore=x' just do lint go licenses +# +# GOROOT is passed explicitly because go-licenses recognizes stdlib packages by +# their location under GOROOT — but, being a go_install binary, its built-in +# default is the GOROOT of whatever toolchain COMPILED it. When that differs +# from the pinned toolchain on PATH (which is where packages actually load +# from), every stdlib package degrades to "does not have module info" errors. +licenses: (_banner "lint go" "licenses") (_per-goos 'GOROOT="$(go env GOROOT)" go-licenses check --include_tests --allowed_licenses=Apache-2.0,BSD-2-Clause,BSD-3-Clause,MIT ${LINT_GO_LICENSES_FLAGS:-} ./...') + +# Bounds-check-elimination report: every bounds check the compiler could NOT +# eliminate, for performance tuning. Informational — it never fails, so it is +# not in the default set; and it reports for the native architecture only, +# since BCE results are arch-specific and this is a report for the human +# reading it, not a gate. +bce: (_banner "lint go" "bce") + #!/usr/bin/env bash + set -euo pipefail + echo "Bounds Check Elimination Report" + echo "================================" + echo "" + output=$(go build -gcflags='-d=ssa/check_bce/debug=1' ./... 2>&1 | grep -v '^#' || true) + if [ -z "$output" ]; then + echo "No bounds checks detected (BCE fully eliminated)." + else + total=$(echo "$output" | wc -l | tr -d ' ') + echo "Total: $total bounds checks" + echo "" + echo "By file:" + echo "$output" | sed 's/:.*$//' | sort | uniq -c | sort -rn + echo "" + echo "Details:" + echo "$output" | sort + fi + +# Escape-analysis and inlining report — both come from the same compiler pass +# (-gcflags=-m): values the compiler moved to the heap (the prime source of GC +# pressure) and functions it refused to inline, with the reason. Informational +# like bce: never fails, native architecture only, not in the default set. +escape: (_banner "lint go" "escape") + #!/usr/bin/env bash + set -euo pipefail + output=$(go build -gcflags='-m' ./... 2>&1 | grep -v '^#' || true) + heap=$(echo "$output" | grep -E 'escapes to heap|moved to heap' || true) + noinline=$(echo "$output" | grep 'cannot inline' || true) + echo "Escape Analysis Report" + echo "======================" + if [ -z "$heap" ]; then + echo "No heap escapes." + else + echo "Total: $(echo "$heap" | wc -l | tr -d ' ') heap escapes" + echo "" + echo "By file:" + echo "$heap" | sed 's/:.*$//' | sort | uniq -c | sort -rn + echo "" + echo "$heap" | sort + fi + echo "" + echo "Inlining Failures" + echo "=================" + if [ -z "$noinline" ]; then + echo "None." + else + echo "Total: $(echo "$noinline" | wc -l | tr -d ' ') functions the compiler could not inline" + echo "" + echo "$noinline" | sort + fi + +# Unreachable functions, by whole-program call-graph analysis (deeper than the +# per-package `unused` linter in golangci). -test keeps library-only repos +# analyzable (test binaries serve as roots) and stops dead code from hiding +# behind test-only callers. Informational: native only, not in the default set. +deadcode: (_banner "lint go" "deadcode") + deadcode -test ./... diff --git a/.limen/just/lint-homebrew.just b/.limen/just/lint-homebrew.just new file mode 100644 index 0000000..7e19c5d --- /dev/null +++ b/.limen/just/lint-homebrew.just @@ -0,0 +1,145 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +# Homebrew formula linting runs through brew itself: brew vendors its own Ruby +# and its own RuboCop with the Homebrew cops, so this module needs no Ruby +# toolchain — installing rubocop separately would be the wrong setup (plain +# RuboCop does not know the formula audit rules). +# +# brew is the one tool here that is NOT on the hermetic PATH, deliberately: +# it is a machine-layer package manager (not aqua-pinnable), and the PATH +# exclusion exists to stop machine tools substituting for pinned ones — brew +# is not substituting for anything, it IS the subject under test; formulas +# only mean anything on a machine that has it. The exception stays narrow: +# BREW_BIN is captured from the AMBIENT PATH at just startup (see main.just), +# before the hermetic PATH locks down — the invoking shell knows where brew +# lives, whatever the prefix — and the hermetic PATH itself is untouched. + +default: style audit + +# Homebrew's RuboCop profile over every formula and cask in the repository; +# passes vacuously when there are none. The mutating twin is +# `just do fix homebrew style`. +style: (_banner "lint homebrew" "style") + #!/usr/bin/env bash + set -euo pipefail + {{ _skip_unless_macos }} + # --cached --others --exclude-standard: tracked *and* new untracked files, + # still honoring .gitignore (same as `lint just`). The patterns are the + # modern tap layout ONLY: Formula/ (sharded subdirectories included) and + # Casks/. The legacy layouts brew still reads (HomebrewFormula/, bare *.rb + # at the repository root) are deliberately unsupported — a formula living + # there is invisible here, so put it in Formula/. Discovery runs before + # the brew lookup so a repo without formulas passes vacuously even + # brew-less. + files=() + while IFS= read -r -d '' f; do + # Tracked-but-deleted files are skipped (see lint.just's `just` recipe). + [ -f "$f" ] || continue + files+=("$f") + done < <(git ls-files -z --cached --others --exclude-standard \ + 'Formula/*.rb' 'Formula/**/*.rb' 'Casks/*.rb' 'Casks/**/*.rb') + if [ "${#files[@]}" -eq 0 ]; then + echo "no formulas or casks (Formula/, Casks/) — nothing to lint." + exit 0 + fi + {{ _require_brew }} + # shellcheck disable=SC2154 # BREW_BIN is exported by the canonical Justfile (main.just). + "$BREW_BIN" style "${files[@]}" + +# The semantic checker: URL/sha256 coherence, license tags, deprecated DSL, +# dependency ordering. brew addresses formulas by TAP NAME, never by path, so +# the project must declare which tap it is (export LINT_HOMEBREW_TAP := +# 'user/name' in the root Justfile) and this recipe registers the working tree +# as that tap — a symlink, so the audit judges the working tree, not a clone — +# for the duration of the run. Extra flags (CI wants --online, which does +# network calls) go through LINT_HOMEBREW_AUDIT_FLAGS. +audit: (_banner "lint homebrew" "audit") + #!/usr/bin/env bash + set -euo pipefail + {{ _skip_unless_macos }} + # Same vacuous pass as `style`: no formulas, nothing to audit, no brew + # (and no tap declaration) required. Tracked-but-deleted files are skipped + # (see lint.just's `just` recipe), so a tree whose only formulas are + # unstaged deletions passes vacuously too. + present="" + while IFS= read -r -d '' f; do + [ -f "$f" ] && present=1 && break + done < <(git ls-files -z --cached --others --exclude-standard \ + 'Formula/*.rb' 'Formula/**/*.rb' 'Casks/*.rb' 'Casks/**/*.rb') + if [ -z "$present" ]; then + echo "no formulas or casks (Formula/, Casks/) — nothing to audit." + exit 0 + fi + {{ _require_brew }} + tap="${LINT_HOMEBREW_TAP:-}" + case "$tap" in + */*) ;; + *) + echo "LINT_HOMEBREW_TAP is not set (or not 'user/name') — brew audit addresses formulas" >&2 + echo "by tap name, not by path. Export it in the root Justfile, e.g.:" >&2 + echo " export LINT_HOMEBREW_TAP := 'farcloser/tap'" >&2 + exit 1 + ;; + esac + # shellcheck disable=SC2154 # BREW_BIN is exported by the canonical Justfile (main.just). + link="$("$BREW_BIN" --repository)/Library/Taps/${tap%%/*}/homebrew-${tap##*/}" + created="" + if [ -e "$link" ] || [ -L "$link" ]; then + # Already tapped: proceed only when it is exactly this checkout — + # auditing a stale clone of the same tap would judge the wrong tree. + if [ "$(readlink "$link" 2>/dev/null || true)" != "$PWD" ]; then + echo "tap $tap is already installed at" >&2 + echo " $link" >&2 + echo "and is not this checkout — untap it (brew untap $tap) or audit there." >&2 + exit 1 + fi + else + mkdir -p "$(dirname "$link")" + ln -s "$PWD" "$link" + created="$link" + fi + cleanup() { + if [ -n "$created" ]; then + rm -f "$created" + fi + } + trap cleanup EXIT + # shellcheck disable=SC2086 # deliberate word-split of the flags knob. + "$BREW_BIN" audit --strict --tap "$tap" ${LINT_HOMEBREW_AUDIT_FLAGS:-} + +# Homebrew exists on macOS only: everywhere else these recipes degrade to a +# LOUD no-op, so a repository can keep homebrew in its lint/fix aggregates +# and still run the full suite on the linux/windows legs of the matrix. +# Injected into recipe bodies as a just variable (same mechanism as +# _require_brew below); the twin lives in fix-homebrew.just. +[private] +_skip_unless_macos := ''' + if [ "$(uname -s)" != 'Darwin' ]; then + echo "homebrew is macOS-only — nothing to do on this platform (skipped, not failed)." + exit 0 + fi +''' + +# BREW_BIN is captured from the ambient PATH in main.just (see the module +# comment); this only refuses when that capture came up empty. Injected into +# recipe bodies as a just variable because module recipes cannot share shell +# functions. +[private] +_require_brew := ''' + if [ -z "${BREW_BIN:-}" ]; then + echo "brew was not found on your PATH when just started." >&2 + echo "brew is a machine-layer tool — install it outside the repo (or export" >&2 + echo "BREW_BIN=/path/to/brew), then re-run." >&2 + exit 1 + fi +''' diff --git a/.limen/just/lint-rust.just b/.limen/just/lint-rust.just new file mode 100644 index 0000000..f7dd113 --- /dev/null +++ b/.limen/just/lint-rust.just @@ -0,0 +1,15 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +rust: (_banner "lint" "rust") + cargo fmt --all --check + cargo clippy --all-targets --all-features -- --deny warnings diff --git a/.limen/just/lint.just b/.limen/just/lint.just new file mode 100644 index 0000000..faea81f --- /dev/null +++ b/.limen/just/lint.just @@ -0,0 +1,263 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +default: limen just aqua links yaml shell dockerfile commits + +# Go linters live in their own submodule: `just do lint go` runs them all, +# `just do lint go ` (e.g. `just do lint go mod`) runs one. +mod go 'lint-go.just' +mod rust 'lint-rust.just' + +# Homebrew formula linting (`just do lint homebrew`): brew's own vendored +# tooling, resolved by absolute path — the machine-layer exception documented +# in the module itself. Explicit like the language modules, not in the +# default set. +mod homebrew 'lint-homebrew.just' + +# limen itself: verify the repository against the Farcloser engineering rules +# (mandatory files, canonical baseline, license, aqua pins). First in the +# default set: the other linters trust the canonical files this one verifies. +# LIMEN_BIN exists for one consumer — the limen repository itself, whose +# root Justfile points it at `go run ./cmd/limen` so the working tree is judged +# by its own enforcer, not by the (always older) released pin. Everyone else +# runs the aqua-pinned binary. +limen: (_banner "lint" "limen") + ${LIMEN_BIN:-limen} check . + +# GitHub settings audit — `limen github check` against this repository +# (inferred from origin), or any target the subcommand takes +# (`just do lint github -org farcloser`). Named, never in the default set: it +# needs the network and an authed gh, and its subject is the live GitHub +# state, not the tree. Same LIMEN_BIN override as `lint limen`. +github *args: (_banner "lint" "github") + ${LIMEN_BIN:-limen} github check {{ args }} + +just: (_banner "lint" "just") + #!/usr/bin/env bash + set -euo pipefail + # --cached --others --exclude-standard: tracked *and* new untracked files, + # still honoring .gitignore — so a not-yet-staged file is linted too. The + # index half of that union also lists files deleted from the worktree but + # not yet staged (a mid-migration tree, e.g. a renamed canonical module), + # so every path is existence-checked before it reaches a tool: enumeration + # is git's (ignore semantics), truth is the working tree's. Deleted files + # are vacuously well-formatted; guarding required files is `limen check`'s + # job, not a formatter's. + # Literal names need their '**/' twins — git pathspec literals do not + # match across directories (only wildcard patterns like '*.just' do) and + # are case-sensitive. All three spellings just honors are covered: + # justfile, Justfile, and the hidden .justfile. + while IFS= read -r -d '' f; do + [ -f "$f" ] || continue + just --fmt --check --justfile "$f" + done < <(git ls-files -z --cached --others --exclude-standard \ + 'justfile' '**/justfile' 'Justfile' '**/Justfile' '.justfile' '**/.justfile' '*.just') + +# godolint — hadolint as pure Go (https://github.com/farcloser/godolint) — +# over every Dockerfile in the repository; passes vacuously when there are +# none, so it can sit in the default set for every repo. Rule exceptions +# belong next to the instruction they excuse, as inline +# `# hadolint ignore=DLxxxx` pragmas — not in the recipe. +dockerfile: (_banner "lint" "dockerfile") + #!/usr/bin/env bash + set -euo pipefail + # --cached --others --exclude-standard: tracked *and* new untracked files, + # still honoring .gitignore — so a not-yet-staged Dockerfile is linted too + # (same as `lint just`). The patterns cover the naming conventions — the + # bare name and the . / . forms, at any + # depth — for both Dockerfile and its OCI-neutral synonym Containerfile + # (podman/buildah). '*.Dockerfile' needs no '**/' twin — a wildcard pattern + # already matches across directories; a literal like 'Dockerfile' does not. + files=() + while IFS= read -r -d '' f; do + # Tracked-but-deleted files are skipped (see `lint just`). + [ -f "$f" ] || continue + files+=("$f") + done < <(git ls-files -z --cached --others --exclude-standard \ + 'Dockerfile' '**/Dockerfile' 'Dockerfile.*' '**/Dockerfile.*' '*.Dockerfile' \ + 'Containerfile' '**/Containerfile' 'Containerfile.*' '**/Containerfile.*' '*.Containerfile') + # godolint validates RUN instructions through shellcheck when it finds it + # on PATH — the hermetic PATH (see the root Justfile) always provides the + # pinned one. + [ ${#files[@]} -eq 0 ] || godolint "${files[@]}" + +yaml: (_banner "lint" "yaml") + yamlfmt -conf .limen/.yamlfmt -lint + +shell: (_banner "lint" "shell") + #!/usr/bin/env bash + set -euo pipefail + # A file is a shell script if it declares itself one, by either signal: + # - a .sh/.bash extension → a missing shebang is then a lint error (SC2148), or + # - a shebang on line 1 for a dialect shellcheck lints (sh/bash/dash/ksh — + # not zsh, which shellcheck cannot check) → covers extension-less scripts. + # Filtering on the shebang alone would silently skip an *.sh that forgot its shebang. + # No mapfile (bash 4+): build the array with a read loop, portable to bash 3.2. + files=() + while IFS= read -r -d '' f; do + # Tracked-but-deleted files are skipped (see `lint just`). + [ -f "$f" ] || continue + case "$f" in + *.sh|*.bash) files+=("$f"); continue ;; + esac + head -n 1 "$f" 2>/dev/null | grep -qE '^#!.*\b(bash|dash|ksh|sh)\b' && files+=("$f") + done < <(git ls-files -z --cached --others --exclude-standard) + [ ${#files[@]} -eq 0 ] || shellcheck --rcfile .limen/.shellcheckrc "${files[@]}" + # Shebang recipes inside justfiles are shell scripts too — extract each + # body from `just --dump --dump-format json` and shellcheck it standalone. + # In the dump a body line is a fragment list: text fragments are strings, + # interpolations (double-brace expressions — just syntax, not shell) are + # arrays, masked as INTERP. One dump of the root justfile covers the whole + # tree: imports (the root Justfile) are flattened into .recipes — they cannot be + # parsed standalone, their dependencies may name root-level module paths — + # and every `mod` nests recursively under .modules. mktemp gets an explicit + # template because macOS mktemp ignores $TMPDIR. + tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/lint-shell.XXXXXX") + trap 'rm -rf "$tmpdir"' EXIT + recipes=() + dump=$(just --dump --dump-format json) + # Bodies are multi-line: base64 carries each across the one-record-per-line + # pipe intact (quotes, tabs and all). The trailing tr matters: the native + # windows jq writes CRLF, and the stray \r lands in the last TSV field, + # corrupting the base64 ("base64: invalid input"). \r cannot legitimately + # appear in this stream (recipe names and base64), so a blanket strip is safe. + while IFS=$'\t' read -r name body; do + # ':' is illegal in windows filenames — the msys layer creates + # "do::lint" via a private-use encoding that native shellcheck then + # cannot open (openBinaryFile: invalid argument). The name is only a + # label; flatten module separators. + out="$tmpdir/${name//:/_}" + printf '%s' "$body" | base64 -d > "$out" + head -n 1 "$out" | grep -qE '^#!.*\b(bash|dash|ksh|sh)\b' || continue + recipes+=("$out") + done < <(printf '%s' "$dump" | jq -r ' + def recipes_of(prefix): + ((.recipes // {}) | to_entries[] + | [prefix + .key, + (.value.body + | map(map(if type == "string" then . else "INTERP" end) | join("")) + | join("\n") | @base64)]), + ((.modules // {}) | to_entries[] + | .key as $mod | .value | recipes_of(prefix + $mod + "::")); + recipes_of("") | @tsv' | tr -d '\r') + # SC1010 is excluded for recipe bodies only: `just do ` is our shared + # namespace, and shellcheck misreads the argument "do" as the shell keyword. + # The constant-word checks (SC2050/SC2078/SC2157/SC2194) are excluded for + # recipe bodies only: a masked INTERP is constant to shellcheck but dynamic + # in reality, so those hits can only be false positives of the masking. + [ ${#recipes[@]} -eq 0 ] || shellcheck --rcfile .limen/.shellcheckrc --exclude=SC1010,SC2050,SC2078,SC2157,SC2194 "${recipes[@]}" + +# Check documentation links with lychee (respects .gitignore, skips hidden files). +links: (_banner "lint" "links") + #!/usr/bin/env bash + set -euo pipefail + # A directory input (not a glob) is what makes lychee honor .gitignore/hidden. + # Canonical exclusions (and their rationale) live in .limen/lychee.toml; a + # repository adds its own in a root .lychee.toml. Both must be passed + # explicitly — any --config disables lychee's ./lychee.toml auto-discovery — + # and lychee merges them, concatenating the exclude lists. + args=(--config .limen/lychee.toml) + if [ -f .lychee.toml ]; then + args+=(--config .lychee.toml) + fi + lychee --no-progress "${args[@]}" . + +# Validate commit hygiene — DCO sign-off, subject length, dangling whitespace — +# over a commit range (default: everything ahead of the upstream default branch; +# pass one explicitly otherwise, e.g. `just do lint commits v1.0.0..HEAD`). +commits range="": (_banner "lint" "commits") + #!/usr/bin/env bash + set -euo pipefail + # Contributors' SSH keys, when the repo ships them, arm signature display + # for humans (git log --show-signature, git tag -v). None of the rules run + # below verify signatures — this is convenience wiring, not enforcement. + # See https://github.com/andyfeller/gh-ssh-allowed-signers for automation + # to retrieve contributor keys. + # Best-effort: sandboxed agent sessions (e.g. Claude Code) write-protect + # .git/config as an escape-vector guard, and losing the display wiring must + # not fail the lint the rules below actually enforce. + if [ -f .allowed_signers ]; then + git config --unset-all gpg.ssh.allowedSignersFile 2>/dev/null || true + git config --add gpg.ssh.allowedSignersFile .allowed_signers 2>/dev/null || + echo "note: .git/config not writable; signature display not armed" >&2 + fi + range="{{ range }}" + if [ -z "$range" ] && [ -n "${GITHUB_BASE_REF:-}" ]; then + # GitHub Actions pull request: validate against the PR's actual base + # branch, whatever it targets. Requires actions/checkout with + # fetch-depth: 0 — the default shallow checkout has no base refs (and + # only the synthetic merge commit as "history": vacuously green). + if git rev-parse --verify -q "origin/$GITHUB_BASE_REF" >/dev/null 2>&1; then + range="origin/${GITHUB_BASE_REF}..HEAD" + fi + fi + if [ -z "$range" ]; then + # Default to the commits this branch adds over the upstream default + # branch. origin/HEAD is only set when `git remote set-head` ran, so + # fall back through the common default-branch names; with no upstream + # at all, validate the full history. + for ref in origin/HEAD origin/main origin/master; do + if git rev-parse --verify -q "$ref" >/dev/null 2>&1; then + range="$ref..HEAD" + break + fi + done + fi + if [ -n "$range" ]; then + git-validation -run DCO,short-subject,dangling-whitespace -range "$range" + else + git-validation -run DCO,short-subject,dangling-whitespace + fi + +# Verify aqua-checksums.json is in sync with aqua.yaml — the working tree's +# pair, exactly as it stands: git state (staged, committed, neither) is +# irrelevant to coherence. Recipes judge the working tree; git answers +# enumeration and history questions only. +aqua: (_banner "lint" "aqua") + #!/usr/bin/env bash + set -euo pipefail + # aqua has no read-only validator, so regenerate, compare against a + # snapshot, and restore (needs network): the lint observes, never mutates — + # `just do fix aqua` is the mutating twin. mktemp gets an explicit template + # because macOS mktemp ignores $TMPDIR. + tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/lint-aqua.XXXXXX") + trap 'rm -rf "$tmpdir"' EXIT + had="" + if [ -f aqua-checksums.json ]; then + cp aqua-checksums.json "$tmpdir/snapshot" + had=1 + fi + # Restore the working tree from the snapshot on EVERY exit — success, drift, + # an aqua failure (set -e), or an interrupt. update-checksum mutates + # aqua-checksums.json in place, and a lint must never leave that behind; an + # inline restore only runs on the paths it is written on, so it belongs in the + # trap. Installed AFTER the snapshot so `had` is known — restoring before that + # would wrongly rm a file we had not yet copied. + # Trap-invoked functions aren't seen as invoked, so this is false-flagged as + # dead code (SC2329) — https://github.com/koalaman/shellcheck/wiki/SC2329. + # shellcheck disable=SC2329 + restore() { + if [ -n "$had" ]; then + cp "$tmpdir/snapshot" aqua-checksums.json + else + rm -f aqua-checksums.json + fi + rm -rf "$tmpdir" + } + trap restore EXIT + # --log-level warn: the per-package INFO lines are noise when nothing is wrong. + aqua --log-level warn update-checksum --prune + if [ -n "$had" ] && cmp -s aqua-checksums.json "$tmpdir/snapshot"; then + exit 0 + fi + echo "aqua-checksums.json is out of sync with aqua.yaml — run 'just do fix aqua'." >&2 + exit 1 diff --git a/.limen/just/main.just b/.limen/just/main.just new file mode 100644 index 0000000..1c2e518 --- /dev/null +++ b/.limen/just/main.just @@ -0,0 +1,104 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# The root Justfile is the project's own: a shim that imports this file and +# then carries the project's recipes — everything shared lives here, so the +# shim never has to change. + +# The project may redefine the shared recipes below (info, or an explicit +# default) in its root Justfile: the later definition — the project's — wins. +set allow-duplicate-recipes +# Likewise for variables: a project's root Justfile may re-export any variable +# set here (notably a hermetic GO* default below) and the importing file wins — +# so an override is explicit and tracked in the Justfile, never an ambient env +# var leaking in. Without this, a duplicate definition is a hard error, which +# is what makes the hermetic defaults below safe to force. +set allow-duplicate-variables + +# Project name, derived from the directory the root Justfile lives in. +project := file_name(justfile_directory()) + +# brew — the one machine-layer tool a shared recipe may call (the homebrew +# modules) — is captured from the AMBIENT PATH here, above the hermetic PATH +# so the lookup still sees the invoking shell's environment: brew is not +# aqua-pinnable and installs at machine-chosen prefixes, so that shell knows +# best where it lives. Empty when absent (the homebrew recipes then fail with +# guidance); override by exporting BREW_BIN. +export BREW_BIN := env_var_or_default('BREW_BIN', `command -v brew || true`) + +# Hermetic PATH: aqua-pinned tools + base system only (no homebrew), so any tool that +# isn't pinned fails loudly instead of silently resolving to an unpinned copy. +# Windows (git-bash) is the sanctioned exception to that hermeticity: there is no +# knowable base-system directory list (sh, coreutils, and git live at +# install-dependent roots), and the separator is ';' — so the pinned tools are +# prepended to the ambient PATH instead. Pins still shadow everything, and +# hermeticity stays enforced by the posix legs of the CI matrix. +aqua_bin := env_var_or_default('AQUA_ROOT_DIR', env_var_or_default('XDG_DATA_HOME', home_directory() / '.local/share') / 'aquaproj-aqua') / 'bin' +export PATH := if os() == 'windows' { aqua_bin + ';' + env_var('PATH') } else { aqua_bin + ":/usr/bin:/bin:/usr/sbin:/sbin" } + +# Hermetic Go env: the PATH is pinned, but Go reads its behavior from a set of +# GO* environment variables that tunnel straight through it — so without this a +# recipe silently inherits the invoking shell's (or an IDE's) Go configuration +# and diverges from CI. Each is emptied or pinned rather than unexported: go +# treats '' as unset, and unlike `unexport` an `export` propagates into module +# recipes. All remain overridable by a project's root Justfile (see +# allow-duplicate-variables above) — an explicit, tracked override, never an +# ambient one. +# +# GOROOT an inherited value (IDEs inject one, often into the module +# cache that `go clean -modcache` deletes) overrides where the +# pinned go finds its stdlib; modern go derives it from its own +# location, so '' is correct. +# GOTOOLCHAIN =local forbids silent toolchain switching: when go.mod +# outpaces the pin, recipes fail loudly asking for a pin bump +# instead of downloading an unpinned toolchain behind your back. +# GOFLAGS ambient flags (-mod=mod, -tags=…, -count=1) rewrite what every +# go command does; '' leaves the recipes' own args the whole story. +# GOSUMDB pinned to the real checksum database so an ambient GOSUMDB=off +# cannot defeat the GOSUMDB verification the go_install pins in +# aqua.yaml depend on. +# GOPRIVATE '' so no module path is silently exempted from the proxy and +# checksum db — a non-empty value is precisely how GOSUMDB gets +# bypassed per-path. +# GOPROXY pinned to the default so an ambient GOPROXY=off or a stale +# mirror cannot change where modules resolve from; a project +# needing an internal proxy sets it in its own Justfile. +# GOOS/GOARCH '' = native. Ambient values silently cross-compile every +# build/lint/test; the per-GOOS analysis sets GOOS itself where +# it needs to (see _per-goos in lib.just), overriding this. +# +# GOWORK is deliberately NOT neutralized: a go.work in the tree — or a parent — +# SHOULD put the recipes into workspace mode. Go workspaces are a supported way +# to work here, the one sanctioned exception to Go-env hermeticity. The cost is +# eyes-open: a build under an active workspace can differ from a CI run (which +# has none), and that divergence is intended, not a leak. +export GOROOT := '' +export GOTOOLCHAIN := 'local' +export GOFLAGS := '' +export GOSUMDB := 'sum.golang.org' +export GOPRIVATE := '' +export GOPROXY := 'https://proxy.golang.org,direct' +export GOOS := '' +export GOARCH := '' + +# Hermetic, per-project linter cache (same doctrine as PATH and GOROOT: no +# shared mutable state across repos). Lives under build/ — gitignored, and +# local runs behave like CI instead of diverging on cache state. +export GOLANGCI_LINT_CACHE := justfile_directory() / 'build/cache/golangci-lint' + +# just's default recipe is the FIRST one defined in the root Justfile, so a +# project's first own recipe takes over from this one — deliberately: the +# default belongs to the project. +# Show every available recipe — the project's own and the shared `do` tree. +default: + @just --list + +# Print meaningful information about this project. +info: + @echo "name: {{ project }}" + @echo "upstream: $(git remote get-url origin 2>/dev/null || echo '(none)')" + @echo "semver: $(git describe --tags --abbrev=0 2>/dev/null || echo '(none)')" + @echo "commit: $(git rev-parse --short HEAD 2>/dev/null || echo '(none)')" + @echo "date: $(git log --max-count=1 --format=%cd --date=short 2>/dev/null || echo '(none)')" + +# Every shared limen task: `just do lint`, `just do build go`, `just do release v1.2.3` — the top level stays the project's. +mod do 'do.just' diff --git a/.limen/just/release.just b/.limen/just/release.just new file mode 100644 index 0000000..203bb45 --- /dev/null +++ b/.limen/just/release.just @@ -0,0 +1,260 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +# Imported flat into the canonical Justfile — not a module — so the recipe can +# take arguments (`just do release v1.2.3`; a module invocation would parse the +# tag as a recipe path). Deliberately no settings and no _lib import here: an +# import's settings would apply to the whole importing module. + +# Release. The default is the CI lane; everything local is opted into with +# --local (see book/recipes.md): +# +# just do release vX.Y.Z +# Verify a clean tree, create the signed tag (your key signs the +# INTENT), push — the release workflow builds, signs the artifacts +# (keyless cosign), and publishes. +# +# just do release --local vX.Y.Z [--cosign-password-stdin] +# The fully local lane: goreleaser + key-based cosign from this machine, +# and the SOLE publisher — it pushes the commit but not the tag, so it does +# not re-trigger the CI release workflow (goreleaser creates the GitHub +# release, and its remote tag, itself). The key path is a mandatory +# argument; the passphrase is read from stdin with --cosign-password-stdin — +# piped, so it never lands in argv or shell history (`pass show cosign | +# just do release … --cosign-password-stdin`) — and prompted on the terminal +# when the flag is absent. For private repos (nothing touches Rekor's public +# log) — and the escape hatch when CI is down. +# +# just do release --local --dry-run +# Unsigned local snapshot into build/release/: no tag, no token, no +# publish — and works on a dirty tree, by design. +# +# just do release --ci +# The workflow half of the default lane (GitHub Actions only): the +# pushed tag triggered us; run goreleaser with keyless signing. +# +# Tag pushes are always exactly one tag, never --tags: stray local tags stay +# local. A tag already on HEAD is reused, so a failed publish retries safely. +[doc('Release: `vX.Y.Z` tags for CI · `--local vX.Y.Z` · `--local --dry-run` · `--ci`')] +release *args: + #!/usr/bin/env bash + set -euo pipefail + # Releasing is opt-in, by carrying a goreleaser config (project-owned, like + # the root Justfile — limen neither pins nor seeds it). Checked before + # anything else: past this point the recipe creates tags and pushes. + if [ ! -f .goreleaser.yaml ] && [ ! -f .goreleaser.yml ]; then + echo "this project has no .goreleaser.yaml — goreleaser releases are opt-in (see book/recipes.md)." >&2 + exit 1 + fi + usage() { + echo "usage: just do release # signed tag + push; CI builds, signs (keyless), publishes" >&2 + echo " just do release --local [--cosign-password-stdin] # fully local, key-based cosign" >&2 + echo " just do release --local --dry-run # local unsigned snapshot into build/release/" >&2 + echo " just do release --ci # workflow half of the default lane (GitHub Actions only)" >&2 + } + require_clean() { + # A dirty tree is the most common failure and goreleaser hard-refuses + # it — check long before anything is created or pushed. + if [ -n "$(git status --porcelain)" ]; then + echo "the working tree is dirty — commit or stash before releasing:" >&2 + git status --short >&2 + exit 1 + fi + } + require_version_tag() { + if ! printf '%s' "$1" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z][0-9A-Za-z.-]*)?$'; then + echo "'$1' is not a version tag (vX.Y.Z or vX.Y.Z-prerelease, e.g. v0.0.0-test.1)." >&2 + exit 2 + fi + } + ensure_signed_tag() { + # Create the signed tag — unless it already exists pointing at HEAD, so + # a retry after a failed publish resumes instead of failing here. + if existing=$(git rev-parse -q --verify "refs/tags/$1^{commit}"); then + if [ "$existing" != "$(git rev-parse HEAD)" ]; then + echo "tag $1 already exists and does not point at HEAD — refusing." >&2 + exit 1 + fi + # Only reuse a pre-existing tag if it is itself SIGNED — otherwise a + # lightweight or unsigned tag placed on HEAD by hand would be pushed, + # silently bypassing the `git tag -s` guarantee. We check that a + # signature block is PRESENT (a lightweight tag has no tag object; + # `git cat-file tag` on it fails and greps empty) rather than running + # `git verify-tag`: verification additionally needs the signer's key + # to be trusted — the GPG keyring, or gpg.ssh.allowedSignersFile for + # SSH — and would reject a legitimately SSH-signed tag when that is + # not set up, breaking the very retry this branch exists for. + if ! git cat-file tag "refs/tags/$1" 2>/dev/null | grep -qE '^-----BEGIN [A-Z0-9]+ SIGNATURE-----'; then + echo "tag $1 is on HEAD but is not a signed tag — refusing to reuse it." >&2 + echo "delete it and let the release create a fresh signed tag: git tag -d $1" >&2 + exit 1 + fi + echo "tag $1 already on HEAD (signed) — reusing it." + else + git tag -s "$1" -m "$1" + fi + } + # The DEFAULT (CI) lane's publish trigger: create the signed tag and push it. + # Pushing a v* tag is what starts the release workflow — this is the ONE event + # that owns publication in the CI lane. The --local lane, by contrast, pushes + # only HEAD (never the tag) so it does NOT re-trigger this workflow: exactly + # one publisher per release. See the --local lane below. + tag_and_push() { + ensure_signed_tag "$1" + git push origin HEAD + git push origin "refs/tags/$1" + } + set -- {{ args }} + case "${1:-}" in + --local) + shift + if [ "${1:-}" = "--dry-run" ]; then + # Snapshot builds skip signing: no key material needed. + exec goreleaser release --snapshot --clean --skip=sign + fi + key="${1:-}" + if [ -z "$key" ]; then + usage + exit 2 + fi + shift + tag="" + password="" + password_from_stdin="" + while [ $# -gt 0 ]; do + case "$1" in + --cosign-password-stdin) + # A boolean flag, NOT a value: the passphrase is read from + # stdin below, so it never enters argv or shell history. + password_from_stdin=1 + shift + ;; + -*) + echo "unknown option: $1" >&2 + usage + exit 2 + ;; + *) + if [ -n "$tag" ]; then + echo "unexpected argument: $1 (tag already given: $tag)" >&2 + usage + exit 2 + fi + tag="$1" + shift + ;; + esac + done + if [ -z "$tag" ]; then + usage + exit 2 + fi + if [ ! -f "$key" ]; then + echo "cosign key not found: $key" >&2 + echo "generate a key pair once with 'cosign generate-key-pair'; commit cosign.pub, keep the private key out of the tree." >&2 + exit 1 + fi + require_version_tag "$tag" + require_clean + if [ -z "${GITHUB_TOKEN:-}" ]; then + echo "GITHUB_TOKEN is not set — goreleaser needs it to publish the GitHub release." >&2 + exit 1 + fi + # cosign under goreleaser cannot prompt (no tty on its stdin), so the + # passphrase is collected here and handed down via COSIGN_PASSWORD. + # That env hand-off to the goreleaser→cosign subtree is unavoidable — + # cosign has no other non-interactive channel — so the goal here is to + # keep the passphrase out of argv/history and out of the caller's own + # shell env: read it once, scoped to this run. Empty is legal (an + # unencrypted key). + if [ -n "$password_from_stdin" ]; then + # --cosign-password-stdin: the passphrase is piped in, so `cat` + # would echo it in cleartext if stdin were the terminal. Refuse + # that footgun and point at the two safe forms. + if [ -t 0 ]; then + echo "--cosign-password-stdin expects the passphrase piped on stdin, but stdin is a terminal." >&2 + echo "pipe it (e.g. 'pass show cosign | just do release --local $key $tag --cosign-password-stdin'), or drop the flag to be prompted." >&2 + exit 2 + fi + # Read ALL of stdin; command substitution strips trailing + # newlines, so a here-string, a `printf | ` pipe, and a file with + # or without a trailing newline all yield the same passphrase. + password=$(cat) + else + # No stdin channel: prompt on the terminal itself (/dev/tty, never + # fd 0 — goreleaser may want stdin, and fd 0 may be redirected). + printf 'cosign key password (empty for an unencrypted key): ' >&2 + read -rs password < /dev/tty + echo >&2 + fi + export COSIGN_KEY="$key" + export COSIGN_PASSWORD="$password" + # Sole-publisher discipline: create the signed tag and push the COMMIT, + # but NOT the tag. goreleaser publishes the GitHub release itself (it + # only needs the tag locally — `validate` checks the working tree, not + # the remote — and creates the release, and its remote tag, through the + # API). Because no `git push` of a v* tag happens, the release workflow + # (on: push tags) never fires, so this machine is the only publisher — + # no race with a second, CI-driven GoReleaser. The signed tag stays + # local (the local lane is for private repos / a CI-down escape hatch); + # the default lane is the one that puts a signed tag on the remote. + ensure_signed_tag "$tag" + git push origin HEAD + exec goreleaser release --clean + ;; + --ci) + # The workflow half: the tag exists (its push triggered this run, + # and CI checks out exactly that commit), publishing uses the + # workflow's GITHUB_TOKEN, and signing is keyless — the OIDC + # identity replaces any key. + if [ -z "${GITHUB_ACTIONS:-}" ]; then + echo "--ci is the workflow half — locally, 'just do release ' cuts the tag CI releases." >&2 + exit 1 + fi + if [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then + echo "no OIDC token available — the workflow needs 'id-token: write' for keyless signing." >&2 + exit 1 + fi + if [ -z "${GITHUB_TOKEN:-}" ]; then + echo "GITHUB_TOKEN is not set — pass the workflow token to this step's env." >&2 + exit 1 + fi + if ! git describe --exact-match --tags HEAD >/dev/null 2>&1; then + echo "HEAD carries no tag — this lane only runs on a tag push." >&2 + exit 1 + fi + # Keyless is enforced, not defaulted: a key leaking into the CI + # environment must not silently flip the signing mode. + unset COSIGN_KEY + exec goreleaser release --clean + ;; + --dry-run) + echo "dry runs are a local concern: use 'just do release --local --dry-run'." >&2 + exit 2 + ;; + "") + usage + exit 2 + ;; + -*) + echo "unknown option: $1" >&2 + usage + exit 2 + ;; + *) + # The default lane: cut the signed tag and push — the release + # workflow takes it from here. + tag="$1" + shift + if [ $# -gt 0 ]; then + echo "unexpected argument: $1" >&2 + usage + exit 2 + fi + require_version_tag "$tag" + require_clean + tag_and_push "$tag" + echo "tag $tag pushed — the release workflow builds, signs, and publishes from here." + ;; + esac diff --git a/.limen/just/test-go.just b/.limen/just/test-go.just new file mode 100644 index 0000000..2d246c1 --- /dev/null +++ b/.limen/just/test-go.just @@ -0,0 +1,140 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +default: unit + +# `go test` runs funnel through gotestsum: same exit semantics, readable live +# output (its format is gotestsum's own GOTESTSUM_FORMAT knob). -count=1 +# defeats Go's test result cache — a test task must actually test. The shared +# timeout is project-overridable from the root Justfile, the home of all project +# customization: +# export TEST_GO_TIMEOUT := '20m' +# +# A PASSING test's output is not shown by default — neither t.Log nor a direct +# write to os.Stderr — so adding prints to chase a green-here/red-in-CI failure +# looks like the prints never ran. Two independent layers suppress it: `go test` +# discards a passing package's output entirely without -v, and gotestsum (which +# passes -json, so it always HAS the output) hides it in its default format. To +# see it: +# GOTESTSUM_FORMAT=standard-verbose just do test go +# +# Every recipe resolves CGO_ENABLED through lib.just's `go_cgo`, exactly as the +# build module does, so a project declaring `export GO_CGO := '1'` TESTS the +# artifact it ships. Left to go's own default this would be silently +# environment-dependent — cgo is on for a native build only where a C toolchain +# happens to be installed, so the same tree would test a cgo binary on a machine +# with Xcode and a pure-Go one without it. `race` is the exception and forces 1 +# unconditionally: the race detector injects runtime/cgo regardless. + +# Unit tests. +unit: (_banner "test go" "unit") + CGO_ENABLED="{{ go_cgo }}" \ + gotestsum -- -count=1 -timeout "${TEST_GO_TIMEOUT:-10m}" ./... + +# Unit tests under the race detector. The final link is delegated to the +# system linker: the race detector injects runtime/cgo into every package, and +# hardened CGO_CFLAGS (-fstack-protector-strong, -fPIE) make those objects +# reference libc symbols Go's internal linker cannot resolve on Linux +# ("relocation target stderr not defined") — see golang/go#52690, #54313, +# #58619. On macOS with Xcode 15+, ld emits "has malformed LC_DYSYMTAB" +# warnings for race builds; cosmetic, the binaries are correct (golang/go#61229). +race: (_banner "test go" "race") + CGO_ENABLED=1 gotestsum -- -count=1 -timeout "${TEST_GO_TIMEOUT:-10m}" -ldflags=-linkmode=external -race ./... + +# Benchmarks, with allocation stats. -run '^$' deselects unit tests so only +# benchmarks run — `just do test go` already covers the tests themselves. +bench: (_banner "test go" "bench") + CGO_ENABLED="{{ go_cgo }}" \ + go test -count=1 -timeout "${TEST_GO_TIMEOUT:-10m}" -run '^$' -bench . -benchmem ./... + +# Coverage: per-function summary, an HTML report under build/coverage/, and an +# optional minimum gate — export TEST_GO_COVER_MIN := '80' (integer percent) +# from the root Justfile to enforce a floor; unset or 0 reports without gating. +cover: (_banner "test go" "cover") + #!/usr/bin/env bash + set -euo pipefail + export CGO_ENABLED="{{ go_cgo }}" + dir=build/coverage + mkdir -p "$dir" + gotestsum -- -count=1 -timeout "${TEST_GO_TIMEOUT:-10m}" -coverprofile="$dir/coverage.out" ./... + go tool cover -func="$dir/coverage.out" + go tool cover -html="$dir/coverage.out" -o "$dir/coverage.html" + echo "HTML report: $dir/coverage.html" + min="${TEST_GO_COVER_MIN:-0}" + # Fail CLOSED on a malformed threshold. `[ "$min" -gt 0 ]` errors on a + # non-integer (a float like 80.0, a typo like 8O), and swallowing that error + # would silently skip the whole gate — a mistyped minimum must not disable + # enforcement. Reject anything that is not a whole number, loudly. + case "$min" in + ''|*[!0-9]*) + echo "TEST_GO_COVER_MIN must be a whole-number percentage (got '$min')." >&2 + exit 2 + ;; + esac + if [ "$min" -gt 0 ]; then + total=$(go tool cover -func="$dir/coverage.out" | awk '/^total:/ { gsub(/%/, "", $3); print $3 }') + if [ "${total%%.*}" -lt "$min" ]; then + echo "coverage ${total}% is below the ${min}% minimum" >&2 + exit 1 + fi + echo "coverage ${total}% meets the ${min}% minimum" + fi + +# CPU and memory profiles, one pair per package, with pprof top-20 summaries +# printed and PNG call graphs rendered via the pinned `dot` (goccy/go-graphviz: +# real graphviz compiled to WASM, so it lives in aqua like any Go tool). +# Informational: a package whose tests fail is still profiled past, never +# fails the run. Artifacts land under build/profiles/ (raw) and +# build/profiles-docs/ (PNG); analyze interactively with `go tool pprof`. +profile: (_banner "test go" "profile") + #!/usr/bin/env bash + set -euo pipefail + export CGO_ENABLED="{{ go_cgo }}" + prof=build/profiles + docs=build/profiles-docs + mkdir -p "$prof" "$docs" + # pprof's own -png shells out to dot writing to a pipe, which the WASM dot + # cannot do (it needs -o) — so render from pprof's -dot text instead. The + # pre-create matters: under a sandbox the WASM runtime can overwrite files + # but not always create them. Paths stay cwd-relative for the same reason + # (the runtime preopens only the working directory). + render() { + src=$1; png=$2; shift 2 + go tool pprof -dot -nodecount=20 "$@" "$src" > "$png.dot" 2>/dev/null + : > "$png" + dot -Tpng -o "$png" "$png.dot" + rm -f "$png.dot" + echo " -> $png" + } + for pkg in $(go list ./...); do + # Filename-safe name from the FULL import path, not just the last segment: + # example/a/client and example/b/client both end in "client" and would + # otherwise overwrite each other's binary and profiles. '/' -> '__'; the + # only other char a Go import path carries that matters here is '.', which + # is already filename-safe. + name=${pkg//\//__} + echo "Profiling $pkg..." + go test -count=1 -o "$prof/$name.test" "$pkg" \ + -cpuprofile "$prof/${name}_cpu.prof" \ + -memprofile "$prof/${name}_mem.prof" || true + if [ -s "$prof/${name}_cpu.prof" ]; then + echo " CPU profile (top 20):" + go tool pprof -top -nodecount=20 "$prof/${name}_cpu.prof" 2>/dev/null || true + render "$prof/${name}_cpu.prof" "$docs/${name}_cpu.png" + fi + if [ -s "$prof/${name}_mem.prof" ]; then + echo " Memory profile — alloc_space (top 20):" + go tool pprof -top -nodecount=20 -alloc_space "$prof/${name}_mem.prof" 2>/dev/null || true + render "$prof/${name}_mem.prof" "$docs/${name}_alloc.png" -alloc_space + fi + done + echo "Profiles written to $prof/, diagrams to $docs/" diff --git a/.limen/just/test.just b/.limen/just/test.just new file mode 100644 index 0000000..94534aa --- /dev/null +++ b/.limen/just/test.just @@ -0,0 +1,17 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +# Go tests live in their own submodule: `just do test go` runs the unit tests, +# `just do test go ` (e.g. `just do test go race`) runs one task. +# +# Bare `just do test` deliberately refuses: tests are always language-bound, so +# there is nothing a universal default could run that would not fail (or lie) +# on some repo. Which suites apply is each project's call — name one +# explicitly, and define the project-wide aggregate as `test` in +# the root Justfile (the CI workflow's entry point, mirroring `lint`). +default: + @echo 'just test has no default — name a suite (e.g. `just do test go`), or run the project'"'"'s `just test`.' >&2 + @exit 1 + +mod go 'test-go.just' diff --git a/.limen/just/tools.just b/.limen/just/tools.just new file mode 100644 index 0000000..a3f44d5 --- /dev/null +++ b/.limen/just/tools.just @@ -0,0 +1,108 @@ +# DO NOT EDIT MANUALLY. +# This file provides shared tasks common to all projects and managed by limen. +# Project recipes live in the root Justfile, below the shared-baseline import. + +set working-directory := '../..' + +# Silence just's per-line command echo; recipes announce themselves via _banner. +set quiet + +# Shared private _banner recipe (see lib.just). +import 'lib.just' + +# --- Project tooling via aqua (see book/tooling.md). Each recipe takes the +# owner/repo exactly as it appears in aqua.yaml, e.g. golangci/golangci-lint. --- +# +# The mutating recipes end with a FULL `aqua install`, never `--only-link`: +# links verify nothing (lazy pulls are a CI economy — wrong at the moment a +# pin changes), and for checksum-less package types (go_install) the +# update-checksum step is silent too, so a broken pin would exit green and +# detonate at first tool use (e.g. a nested, untagged Go module pinned to a +# repo tag its module zip does not contain). On a warm machine the full +# install is incremental — only the touched tool downloads or builds. + +# Add a new tool at its latest version, e.g. `just do tools add junegunn/fzf`. +add pkg: (_banner "tools" "add") + #!/usr/bin/env bash + set -euo pipefail + # Regex-escape the package name before it enters a grep -E / sed -E pattern. + # aqua slugs carry dots (github.com, golang.org), and an unescaped '.' is a + # wildcard: without this, `add foo/li.r` would match an existing `foo/liar` + # (and set/remove would edit/delete the wrong entry). ']' and '\' are omitted + # from the class — a Go module slug can contain neither, and both trip BSD sed. + esc=$(printf '%s' '{{ pkg }}' | sed 's/[.^$*+?()[{}|]/\\&/g') + if grep -qE "^[[:space:]]*-[[:space:]]*name:[[:space:]]*${esc}([[:space:]]|@|$)" aqua.yaml; then + echo "{{ pkg }} is already in aqua.yaml — use 'just do tools update {{ pkg }}' or 'just do tools set {{ pkg }} '." >&2 + exit 1 + fi + aqua generate -i "{{ pkg }}" # append the latest version to aqua.yaml + aqua update-checksum --prune # record its checksum (and drop stale ones) + aqua install # REAL install: a bad pin fails here, not at first use + +# Set an existing tool to an exact version, e.g. `just do tools set golangci/golangci-lint v1.55.3`. +set pkg version: (_banner "tools" "set") + #!/usr/bin/env bash + set -euo pipefail + # Regex-escape the slug before grep -E / sed -E — see `add` for why. + esc=$(printf '%s' '{{ pkg }}' | sed 's/[.^$*+?()[{}|]/\\&/g') + if ! grep -qE "^[[:space:]]*-[[:space:]]*name:[[:space:]]*${esc}@" aqua.yaml; then + echo "{{ pkg }} is not pinned in aqua.yaml — add it with 'just do tools add {{ pkg }}'." >&2 + exit 1 + fi + # An explicit path template: macOS mktemp ignores $TMPDIR and falls back to + # a per-user temp dir that sandboxes may deny writes to. A sibling file also + # makes the mv a same-filesystem, truly atomic rename. + tmp=$(mktemp aqua.yaml.XXXXXX) + trap 'rm -f "$tmp"' EXIT + sed -E "s#(^[[:space:]]*-[[:space:]]*name:[[:space:]]*${esc})@[^[:space:]]*#\1@{{ version }}#" aqua.yaml > "$tmp" + mv "$tmp" aqua.yaml + aqua update-checksum --prune # new checksum in, replaced version's out + aqua install # REAL install: a bad pin fails here, not at first use + +# Update an existing tool to its latest version, e.g. `just do tools update golangci-lint`. +# Takes the COMMAND name (the executable you type — `just`, `limen`), not the +# owner/name package slug: it delegates to `aqua update`, which resolves a +# command to its pinned package — local-registry packages included. (The +# previous `aqua generate` approach could not work: generate refuses to +# re-emit an already-pinned package, which is the only kind update meets.) +update command: (_banner "tools" "update") + #!/usr/bin/env bash + set -euo pipefail + aqua update "{{ command }}" # bump aqua.yaml to the latest release + aqua update-checksum --prune # new checksum in, replaced version's out + aqua install # REAL install: a bad pin fails here, not at first use + +# Remove a tool entirely, e.g. `just do tools remove junegunn/fzf`. +remove pkg: (_banner "tools" "remove") + #!/usr/bin/env bash + set -euo pipefail + # Regex-escape the slug before grep -E — see `add` for why. + esc=$(printf '%s' '{{ pkg }}' | sed 's/[.^$*+?()[{}|]/\\&/g') + if ! grep -qE "^[[:space:]]*-[[:space:]]*name:[[:space:]]*${esc}([[:space:]]|@|$)" aqua.yaml; then + echo "{{ pkg }} is not in aqua.yaml — nothing to remove." >&2 + exit 1 + fi + # See `set` for why mktemp gets an explicit sibling-path template. + tmp=$(mktemp aqua.yaml.XXXXXX) + trap 'rm -f "$tmp"' EXIT + # Match by an EXACT string compare on the parsed slug, not a regex built from + # it. A regex would make a dot in the name a wildcard (see `add`) and delete a + # sibling — and escaping for awk is unportable (its -v processing eats the + # backslashes), so we parse the slug out and compare literally instead. Drops + # the entry's line plus its indented continuation lines (registry:, etc.). + awk -v pkg="{{ pkg }}" ' + function isCont(l) { return (l ~ /^[[:blank:]]+[^[:blank:]#-]/) } + { + if (skip) { if (isCont($0)) next; skip = 0 } + if (match($0, /^[[:blank:]]*-[[:blank:]]*name:[[:blank:]]*/)) { + slug = substr($0, RLENGTH + 1) + sub(/[[:blank:]].*$/, "", slug) # drop trailing spaces / comment + sub(/@.*$/, "", slug) # drop @version + if (slug == pkg) { skip = 1; next } + } + print + } + ' aqua.yaml > "$tmp" + mv "$tmp" aqua.yaml + aqua remove "{{ pkg }}" || true # uninstall the binary (no-op for go_install tools) + aqua update-checksum --prune # drop the now-unused checksum diff --git a/.limen/lychee.toml b/.limen/lychee.toml new file mode 100644 index 0000000..3467b41 --- /dev/null +++ b/.limen/lychee.toml @@ -0,0 +1,31 @@ +# DO NOT EDIT MANUALLY. +# Canonical lychee (link checker) configuration, identical in every repository. +# A repository can add its own exclusions in a root .lychee.toml — the `lint links` +# recipe passes both files and lychee merges them, concatenating the exclude +# lists — so this baseline only carries exclusions that apply everywhere. + +# Traverse dotted directories. lychee skips hidden paths by default, which +# silently excludes documentation living under `.github/` — invisible in most +# repositories (their markdown sits at the root or under book/), and total in +# the org's `.github` repository, where every policy document IS in `.github/`. +# A link check that quietly covers nothing is worse than no link check. This is +# independent of .gitignore handling (lychee's separate --no-ignore, which we +# never pass), so build artifacts stay excluded, and `.git` is not walked. +hidden = true + +# Transient failures (5xx, 408, 429) are retried by lychee, but its defaults +# (3 retries, 2s minimum wait) span mere seconds — shorter than a typical +# GitHub 503 blip, so runs failed on weather. Widen the window instead of +# accepting the codes outright: a server that is STILL erroring after ~a +# minute of backoff is a finding, not weather. +max_retries = 6 +retry_wait_time = 5 + +exclude = [ + # fsf.org: verbatim (A)GPL license texts link to it, and its server (TLS 1.2 + # with DHE-only key exchange) cannot complete a handshake with lychee's + # rustls, which implements neither. The links themselves are fine. + # gnu.org: rate-limits aggressively, so the license-text links to it fail + # intermittently. + 'https?://(www\.)?(fsf|gnu)\.org(/|$)', +] diff --git a/Justfile b/Justfile new file mode 100644 index 0000000..d9b33e3 --- /dev/null +++ b/Justfile @@ -0,0 +1,90 @@ +# This file is the project's own. +# Add recipes leveraging provided `do` ready-made recipes, or create your own. +# The import must be kept: it mounts every shared limen task under `just do ...`. +import '.limen/just/main.just' + +# The FIRST recipe defined here becomes `just`'s default. +lint: do::lint::go::default do::lint::default lint-generated +fix: do::fix::go::default do::fix::default +test: simd-info do::test::go::unit do::test::go::race test-386 +bench: do::test::go::bench + +# The amd64 assembly is generated — never edited — and this proves it: the +# committed file must be the exact output of the pinned generator. avo is a +# code-generation dependency of the generator alone, so it lives in its own +# module (avo/go.mod, GOSUMDB-verified like any other) and never appears in +# the main module's graph. +[doc('Verify guts/compress_amd64.s is the exact output of avo/gen.go')] +lint-generated: + #!/usr/bin/env bash + set -euo pipefail + # avo embeds its -out argument in the generated header, so a byte-exact + # comparison must regenerate with the exact command `just gen` runs. + # Snapshot and restore: the lint observes, never mutates (same doctrine as + # `do lint aqua`). + tmp=$(mktemp "${TMPDIR:-/tmp}/blake3-avo.XXXXXX") + # shellcheck disable=SC2329 # invoked via trap, not dead code + restore() { + cp "$tmp" guts/compress_amd64.s + rm -f "$tmp" + } + trap restore EXIT + cp guts/compress_amd64.s "$tmp" + go generate ./guts + if ! cmp -s "$tmp" guts/compress_amd64.s; then + echo "guts/compress_amd64.s does not match the output of avo/gen.go — run 'just gen'" >&2 + exit 1 + fi + +[doc('Regenerate guts/compress_amd64.s from avo/gen.go')] +gen: + go generate ./guts + +# The SIMD kernels are selected by runtime CPU detection, so which +# implementation the suite just exercised is a property of the host. Say so in +# the log: a green run on a runner without AVX-512 is not evidence about the +# AVX-512 code. Diagnostic only — never fails. +[doc('Report which BLAKE3 SIMD paths this host can exercise')] +simd-info: + #!/usr/bin/env bash + set -euo pipefail + arch="$(go env GOHOSTARCH)" + echo "host: $(go env GOHOSTOS)/$arch" + if [ "$arch" != "amd64" ]; then + echo "simd: none (the assembly is amd64-only; this host runs the generic implementation)" + exit 0 + fi + if [ -r /proc/cpuinfo ]; then + for feature in avx2 avx512f; do + if grep -qw "$feature" /proc/cpuinfo; then + echo "simd: $feature available" + else + echo "simd: $feature NOT available" + fi + done + elif command -v sysctl > /dev/null 2>&1; then + for feature in hw.optional.avx2_0 hw.optional.avx512f; do + if [ "$(sysctl -n "$feature" 2> /dev/null)" = "1" ]; then + echo "simd: $feature available" + else + echo "simd: $feature NOT available" + fi + done + else + echo "simd: unknown (no /proc/cpuinfo or sysctl on this host)" + fi + +# 32-bit coverage: the codebase is int-width sensitive (buffer arithmetic, +# uint64 stream offsets against a 32-bit int), and no development machine is +# 32-bit. 386 binaries execute natively on amd64 hosts, so the amd64 CI legs +# run this for free; other hosts skip it loudly. The race detector does not +# support 386. +[doc('Run the tests as GOARCH=386 (executes natively on amd64 hosts; skipped elsewhere)')] +test-386: + #!/usr/bin/env bash + set -euo pipefail + if [ "$(go env GOHOSTARCH)" != "amd64" ]; then + echo "GOARCH=386 binaries need an amd64 host to execute; skipping" + exit 0 + fi + CGO_ENABLED=0 GOARCH=386 go test -count=1 -timeout "${TEST_GO_TIMEOUT:-10m}" ./... diff --git a/README.md b/README.md index 749fb49..bc7db2f 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,8 @@ blake3 ------ -[![GoDoc](https://godoc.org/lukechampine.com/blake3?status.svg)](https://godoc.org/lukechampine.com/blake3) -[![Go Report Card](http://goreportcard.com/badge/lukechampine.com/blake3)](https://goreportcard.com/report/lukechampine.com/blake3) - ``` -go get lukechampine.com/blake3 +go get github.com/forkcloser/blake3 ``` `blake3` implements the [BLAKE3 cryptographic hash function](https://github.com/BLAKE3-team/BLAKE3). diff --git a/aqua-checksums.json b/aqua-checksums.json new file mode 100644 index 0000000..d40eca2 --- /dev/null +++ b/aqua-checksums.json @@ -0,0 +1,284 @@ +{ + "checksums": [ + { + "id": "github_release/github.com/casey/just/1.57.0/just-1.57.0-aarch64-apple-darwin.tar.gz", + "checksum": "0381DB216C2F97CE31D838A1562C1064DFBFA73F5A8A81581338A2CD9217DF47", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/casey/just/1.57.0/just-1.57.0-aarch64-unknown-linux-musl.tar.gz", + "checksum": "F225044A81ADEA6E0B3A8B9370AAF374E6AF76C8735AE263AC993DF55FD137EC", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/casey/just/1.57.0/just-1.57.0-x86_64-pc-windows-msvc.zip", + "checksum": "4C7391D17CB1D17B758B52004EE6411372B8A13FF37C3C9B9031625CB6026E09", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/casey/just/1.57.0/just-1.57.0-x86_64-unknown-linux-musl.tar.gz", + "checksum": "45B548094283CB9739AF8F13273B8CDDEEE869F5B4EF2BB631B1F311CB566155", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/cli/cli/v2.96.0/gh_2.96.0_linux_amd64.tar.gz", + "checksum": "83D5C2CCAD5498F58BF6368ACB1AB32588CF43AB3A4B1C301BF36328B1C8BD60", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/cli/cli/v2.96.0/gh_2.96.0_linux_arm64.tar.gz", + "checksum": "06F86EC7103D41993B76CD78072F43595C34AAA56506D971D9860E67140BF909", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/cli/cli/v2.96.0/gh_2.96.0_macOS_arm64.zip", + "checksum": "F23A0C37D963AACC3BED703CCBD59B41C5CA22101FAB7F00EB2B7CAD23ABA463", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/cli/cli/v2.96.0/gh_2.96.0_windows_amd64.zip", + "checksum": "C2D6ACC935CD2F00E2144D7E036D5CD82E6B6BD5594E8C75AA75EF2A4ED6AAC3", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/cli/cli/v2.96.0/gh_2.96.0_windows_arm64.zip", + "checksum": "C517E0B32C98A4BA90AC95AF8D12CC3AC55781AB4AB72F9A91CE3DE0541D2B09", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/farcloser/limen/v0.0.10/limen_0.0.10_darwin_arm64.tar.gz", + "checksum": "C5F7CB59CA8313A96D80597E32A2F5CEBDD2D094D6E30A9FC62F5AE0CA661584", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/farcloser/limen/v0.0.10/limen_0.0.10_linux_amd64.tar.gz", + "checksum": "F4D005A929CDA0678946054FC11FF79862A96653818F0E8C1813E583FC2F9626", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/farcloser/limen/v0.0.10/limen_0.0.10_linux_arm64.tar.gz", + "checksum": "7F1090310543E9AB9D83A4AEFC6F47C41CA047B080BAC2F9F18873A1C79A294C", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/farcloser/limen/v0.0.10/limen_0.0.10_windows_amd64.tar.gz", + "checksum": "3F78DB94E076C7AE6E102AF70B60691D5C79C29948469390BEE8326758A09957", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/farcloser/limen/v0.0.10/limen_0.0.10_windows_arm64.tar.gz", + "checksum": "DD5A8AA4DB208D88F7BE93CE92A6833F5A74C416599177CE3744821C108FEAC5", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/golangci/golangci-lint/v2.12.2/golangci-lint-2.12.2-darwin-arm64.tar.gz", + "checksum": "A9C54498731B3128F79E090BE6110F3E5FFFCCC617B08142ED244D4126C73F29", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/golangci/golangci-lint/v2.12.2/golangci-lint-2.12.2-linux-amd64.tar.gz", + "checksum": "8DF580D2670FED8FA984AAC0507099AF8DF275E665215F5C7A2AE3943893A553", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/golangci/golangci-lint/v2.12.2/golangci-lint-2.12.2-linux-arm64.tar.gz", + "checksum": "44CD40A8C76C86755375ADFEEA52CFD3533CB43D7BD647771E0AE065E166DF3A", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/golangci/golangci-lint/v2.12.2/golangci-lint-2.12.2-windows-amd64.zip", + "checksum": "BD42E3EBC8CB4ECECB86941983BAAF1DC221BBB04D838E94CE63B49CC91E02BB", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/golangci/golangci-lint/v2.12.2/golangci-lint-2.12.2-windows-arm64.zip", + "checksum": "947B9A5BF762D465710B376C156F0184ABB2168378B0826AF1899E0EE7183742", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/google/yamlfmt/v0.21.0/yamlfmt_0.21.0_Darwin_arm64.tar.gz", + "checksum": "4B417ECB94339D57E4C122ECC948C1A00FE328B5853266DE9806E652A92858FA", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/google/yamlfmt/v0.21.0/yamlfmt_0.21.0_Linux_arm64.tar.gz", + "checksum": "5B2689C963B177271330C5CE8CA7396751107E5A826BE46F03D2CB9B6F0C7784", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/google/yamlfmt/v0.21.0/yamlfmt_0.21.0_Linux_x86_64.tar.gz", + "checksum": "1F300D9257B232BB3B541D7FB1B0E6B3C121BCBAB381C86CD38CB8722BE8A566", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/google/yamlfmt/v0.21.0/yamlfmt_0.21.0_Windows_arm64.tar.gz", + "checksum": "C1E64D1C72CA8986BC5B8C8EDD4EC89F0627804E7E08F8DE9F4B484CB5CAD897", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/google/yamlfmt/v0.21.0/yamlfmt_0.21.0_Windows_x86_64.tar.gz", + "checksum": "07F80CE5D741EB4B0A9380AC78A19C7CB5BD44E2A9A47A5A04839E3BA54DD463", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/goreleaser/goreleaser/v2.17.1/goreleaser_Darwin_all.tar.gz", + "checksum": "F49D4FE67D283B5B5130C380C983087DCA0A4D4EC15B6637B18FE1AB096780D8", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/goreleaser/goreleaser/v2.17.1/goreleaser_Linux_arm64.tar.gz", + "checksum": "702F03769AC8BCB0E47839C82243CC614AE995633599A98C63062E13EA85F829", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/goreleaser/goreleaser/v2.17.1/goreleaser_Linux_x86_64.tar.gz", + "checksum": "A99BBC7AE0D8D897B07C4C497A9B62F222558804715EF219D1AF05A7E417BC80", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/goreleaser/goreleaser/v2.17.1/goreleaser_Windows_arm64.zip", + "checksum": "6EEC917C98DF13BA83B8BB9C261E017DF2D1D5B08E525442B5918325627FDF91", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/goreleaser/goreleaser/v2.17.1/goreleaser_Windows_x86_64.zip", + "checksum": "53314CE7CC16C3229F2D3B98A932C1618C427964EB2170A1EFAFBDFA862A556F", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/gotestyourself/gotestsum/v1.13.0/gotestsum_1.13.0_darwin_arm64.tar.gz", + "checksum": "509CB27AEF747F48FAF9BCE424F59DCF79572C905204B990EE935BBFCC7FA0E9", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/gotestyourself/gotestsum/v1.13.0/gotestsum_1.13.0_linux_amd64.tar.gz", + "checksum": "11CCDDEAF708EF228889F9FE2F68291A75B27013DDFC3B18156E094F5F40E8EE", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/gotestyourself/gotestsum/v1.13.0/gotestsum_1.13.0_linux_arm64.tar.gz", + "checksum": "7644A4C5CD1BB978D56245AEAB25A586AC5AC62ADEBED20A399548867C13499D", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/gotestyourself/gotestsum/v1.13.0/gotestsum_1.13.0_windows_amd64.tar.gz", + "checksum": "FD5A6DC69E46A0970593E70D85A7E75F16714E9C61D6D72CCC324EB82DF5BB8A", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/gotestyourself/gotestsum/v1.13.0/gotestsum_1.13.0_windows_arm64.tar.gz", + "checksum": "72A59200F83B3204CD59FD417E384DB0543C5511D7A9E38957E74A4035950943", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/jqlang/jq/jq-1.8.2/jq-linux-amd64", + "checksum": "B1C22172DD303F3BE49E935AA56AA48A8B7A46E0BC838B4997D3BB451495870F", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/jqlang/jq/jq-1.8.2/jq-linux-arm64", + "checksum": "8B85C817833814DDCA00A144C33705546355AFCCF0CF39B188F3CDB48B852309", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/jqlang/jq/jq-1.8.2/jq-macos-arm64", + "checksum": "2D75340BA57A4B4B4C8708A21C2DC8E958A48AAA8BBA13B27F77F6E4C0ECA07E", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/jqlang/jq/jq-1.8.2/jq-windows-amd64.exe", + "checksum": "A6FC67FEDAF9128A3309A1E2EBB8B986AECCF70122EE46D2CB4849E423F0C627", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/koalaman/shellcheck/v0.11.0/shellcheck-v0.11.0.darwin.aarch64.tar.xz", + "checksum": "56AFFDD8DE5527894DCA6DC3D7E0A99A873B0F004D7AABC30AE407D3F48B0A79", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/koalaman/shellcheck/v0.11.0/shellcheck-v0.11.0.linux.aarch64.tar.xz", + "checksum": "12B331C1D2DB6B9EB13CFCA64306B1B157A86EB69DB83023E261EAA7E7C14588", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/koalaman/shellcheck/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz", + "checksum": "8C3BE12B05D5C177A04C29E3C78CE89AC86F1595681CAB149B65B97C4E227198", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/koalaman/shellcheck/v0.11.0/shellcheck-v0.11.0.zip", + "checksum": "8A4E35AB0B331C85D73567B12F2A444DF187F483E5079CEFFA6BDA1FAA2E740E", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/lycheeverse/lychee/lychee-v0.24.2/lychee-aarch64-apple-darwin.tar.gz", + "checksum": "C9D3740EA2D891854D37116C9FBA840F37B6E7C89D330E7DB84AC333631C4977", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/lycheeverse/lychee/lychee-v0.24.2/lychee-aarch64-unknown-linux-musl.tar.gz", + "checksum": "5D0B0E3AEAB240F41920C633A6EAF97599BE6EEDDA034B36E858EDE7DBA5E535", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/lycheeverse/lychee/lychee-v0.24.2/lychee-x86_64-pc-windows-msvc.zip", + "checksum": "32975D1493EE1A975D6BB41E4FB56FE419CB442DED628BB772BA2E614ACFACAD", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/lycheeverse/lychee/lychee-v0.24.2/lychee-x86_64-unknown-linux-musl.tar.gz", + "checksum": "73657A111819A30C47C08352896796F23D64E4EB2B3ED39B6D32149241566FC5", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/sigstore/cosign/v3.1.1/cosign-darwin-arm64", + "checksum": "94B42A9E697BE95675F6160AB031A9A5F1EC1E646D6F648D7B2F5CD59ECECBC5", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/sigstore/cosign/v3.1.1/cosign-linux-amd64", + "checksum": "AE1ECD212663F3693AD9EDF8B1A183900C9A52D3155BA6E354237F9A0F6463FC", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/sigstore/cosign/v3.1.1/cosign-linux-arm64", + "checksum": "2EC865872E331C32FD12B08DAE15332D3F92C0AA029219589684A4903CA85D11", + "algorithm": "sha256" + }, + { + "id": "github_release/github.com/sigstore/cosign/v3.1.1/cosign-windows-amd64.exe", + "checksum": "9D2C026E667BFD979FA7BA1CAB8C4B24D2E73F336EC2D57F7FC72C7E73E5B4B6", + "algorithm": "sha256" + }, + { + "id": "http/golang.org/dl/go1.26.5.darwin-arm64.tar.gz", + "checksum": "EFB87FF28AF9A188D0536EF5D42E63DD52BA8263CD7344A993CC48DD11DEDB6A", + "algorithm": "sha256" + }, + { + "id": "http/golang.org/dl/go1.26.5.linux-amd64.tar.gz", + "checksum": "5C2C3B16CAEFA1D968A94C1DACA04A7CA301A496D9B086E17AD77BB81393F053", + "algorithm": "sha256" + }, + { + "id": "http/golang.org/dl/go1.26.5.linux-arm64.tar.gz", + "checksum": "FE4789E92B1F33358680864BBE8704289E7BB5FC207D80623C308935BD696D49", + "algorithm": "sha256" + }, + { + "id": "http/golang.org/dl/go1.26.5.windows-amd64.zip", + "checksum": "97E6B2A833B6D89F9FF17D25419AC0A7E3B482A044E9AB18CDEF834BD834FD38", + "algorithm": "sha256" + }, + { + "id": "http/golang.org/dl/go1.26.5.windows-arm64.zip", + "checksum": "F96EE46396D69F1E231C8D981EC6A70216238A646A1F2CD74AEA0D0016BBC017", + "algorithm": "sha256" + }, + { + "id": "registries/github_content/github.com/aquaproj/aqua-registry/v4.544.0/registry.yaml", + "checksum": "ED73709C88FA03BE758F78C25A35C07D48E2FA9D19E407F89FC36989A306E111", + "algorithm": "sha256" + } + ] +} diff --git a/aqua-policy.yaml b/aqua-policy.yaml new file mode 100644 index 0000000..2e4e6b0 --- /dev/null +++ b/aqua-policy.yaml @@ -0,0 +1,13 @@ +# DO NOT EDIT MANUALLY. +# This file is common to all projects and managed by limen. +# Global configuration changes proposals can be discussed on https://github.com/farcloser/limen + +registries: + - type: standard + ref: semver(">= 4.0.0") + - name: local + type: local + path: .limen/aqua-registry.yaml +packages: + - registry: standard + - registry: local diff --git a/aqua.yaml b/aqua.yaml new file mode 100644 index 0000000..1c110ad --- /dev/null +++ b/aqua.yaml @@ -0,0 +1,48 @@ +# aqua — Declarative CLI Version Manager — https://aquaproj.github.io/ +checksum: + enabled: true + require_checksum: true + supported_envs: + - darwin/arm64 + - linux/amd64 + - linux/arm64 + - windows/amd64 + - windows/arm64 + +registries: + - type: standard + ref: v4.544.0 # renovate: depName=aquaproj/aqua-registry + - name: local + type: local + path: .limen/aqua-registry.yaml + +packages: + # --- go install tools (local registry, GOSUMDB-verified) --- + - name: github.com/google/go-licenses/v2@v2.0.1 + registry: local + - name: github.com/vbatts/git-validation@v1.2.2 + registry: local + - name: golang.org/x/vuln/cmd/govulncheck@v1.5.0 + registry: local + - name: golang.org/x/tools/cmd/deadcode@v0.47.0 + registry: local + # Pseudo-version: the nested cmd/dot module carries no tags upstream. + - name: github.com/goccy/go-graphviz/cmd/dot@v0.0.0-20251129032125-76e04975df88 + registry: local + - name: github.com/farcloser/godolint/cmd/godolint@v0.1.0 + registry: local + # --- farcloser tools (local registry; standard once registered upstream) --- + - name: farcloser/limen@v0.0.10 # renovate: depName=farcloser/limen + registry: local + # --- toolchain + binary-release tools (standard registry, aqua-verified) --- + - name: golang/go@go1.26.5 + - name: casey/just@1.57.0 + - name: koalaman/shellcheck@v0.11.0 + - name: golangci/golangci-lint@v2.12.2 + - name: google/yamlfmt@v0.21.0 + - name: lycheeverse/lychee@lychee-v0.24.2 + - name: goreleaser/goreleaser@v2.17.1 + - name: sigstore/cosign@v3.1.1 + - name: gotestyourself/gotestsum@v1.13.0 + - name: jqlang/jq@jq-1.8.2 + - name: cli/cli@v2.96.0 diff --git a/avo/gen.go b/avo/gen.go index 2137c18..a8ae192 100644 --- a/avo/gen.go +++ b/avo/gen.go @@ -1,6 +1,6 @@ -//go:build ignore -// +build ignore - +// Package main generates the amd64 assembly implementations in guts. +// It is its own module so that avo — a code-generation dependency, not a +// library dependency — never appears in the main module's graph. package main import ( diff --git a/avo/go.mod b/avo/go.mod new file mode 100644 index 0000000..3106f1a --- /dev/null +++ b/avo/go.mod @@ -0,0 +1,11 @@ +module github.com/forkcloser/blake3/avo + +go 1.25.0 + +require github.com/mmcloughlin/avo v0.6.0 + +require ( + golang.org/x/mod v0.38.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/tools v0.48.0 // indirect +) diff --git a/avo/go.sum b/avo/go.sum new file mode 100644 index 0000000..724fa88 --- /dev/null +++ b/avo/go.sum @@ -0,0 +1,10 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY= +github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= diff --git a/bao/bao.go b/bao/bao.go index beda376..34be029 100644 --- a/bao/bao.go +++ b/bao/bao.go @@ -8,7 +8,7 @@ import ( "io" "math/bits" - "lukechampine.com/blake3/guts" + "github.com/forkcloser/blake3/guts" ) func bytesToCV(b []byte) (cv [8]uint32) { @@ -28,7 +28,9 @@ func cvToBytes(cv *[8]uint32) *[32]byte { } func compressGroup(p []byte, counter uint64) guts.Node { - var stack [54 - guts.MaxSIMD][8]uint32 + // stack size is log2(maximum number of buffers in a group), i.e. + // log2(2^64 bytes / ChunkSize / MaxSIMD) = 64 - 10 - 4 + var stack [50][8]uint32 var sc uint64 pushSubtree := func(cv [8]uint32) { i := 0 @@ -106,6 +108,9 @@ func Encode(dst io.WriterAt, data io.Reader, dataLen int64, group int, outboard // the I/O required in half, at the cost of making it a lot trickier to hash // multiple groups in SIMD. However, you can still get the SIMD speedup if // group > 0, so maybe just do that. + // parentBuf is reused for all parent nodes; it escapes into dst.WriteAt, + // so a per-node buffer would mean a heap allocation per node + var parentBuf [64]byte var rec func(bufLen uint64, flags uint32, off uint64) (uint64, [8]uint32) rec = func(bufLen uint64, flags uint32, off uint64) (uint64, [8]uint32) { if err != nil { @@ -127,8 +132,11 @@ func Encode(dst io.WriterAt, data io.Reader, dataLen int64, group int, outboard llen += (mid / groupSize) * groupSize } rchildren, r := rec(bufLen-mid, 0, off+64+llen) - write(cvToBytes(&l)[:], off) - write(cvToBytes(&r)[:], off+32) + for i := range l { + binary.LittleEndian.PutUint32(parentBuf[4*i:], l[i]) + binary.LittleEndian.PutUint32(parentBuf[32+4*i:], r[i]) + } + write(parentBuf[:], off) return 2 + lchildren + rchildren, guts.ChainingValue(guts.ParentNode(l, r, &guts.IV, flags)) } @@ -141,6 +149,10 @@ func Encode(dst io.WriterAt, data io.Reader, dataLen int64, group int, outboard // Decode reads content and tree data from the provided reader(s), and // streams the verified content to dst. It returns false if verification fails. // If the content and tree data are interleaved, outboard should be nil. +// +// Decode reads the tree data 64 bytes at a time, so if the readers are +// unbuffered (e.g. os.File), wrapping them in a bufio.Reader will +// significantly improve performance. func Decode(dst io.Writer, data, outboard io.Reader, group int, root [32]byte) (bool, error) { if outboard == nil { outboard = data @@ -231,10 +243,10 @@ func ExtractSlice(dst io.Writer, data, outboard io.Reader, group int, offset uin groupSize := uint64(guts.ChunkSize << group) buf := make([]byte, groupSize) var err error - read := func(r io.Reader, n uint64, copy bool) { + read := func(r io.Reader, n uint64, emit bool) { if err == nil { _, err = io.ReadFull(r, buf[:n]) - if err == nil && copy { + if err == nil && emit { _, err = dst.Write(buf[:n]) } } @@ -257,7 +269,7 @@ func ExtractSlice(dst io.Writer, data, outboard io.Reader, group int, offset uin } read(outboard, 8, true) dataLen := binary.LittleEndian.Uint64(buf[:8]) - if dataLen < offset+length { + if end := offset + length; end < offset || dataLen < end { return errors.New("invalid slice length") } rec(0, dataLen) @@ -267,6 +279,10 @@ func ExtractSlice(dst io.Writer, data, outboard io.Reader, group int, offset uin // DecodeSlice reads from data, which must contain a slice encoding for the // given offset and length, and streams verified content to dst. It returns // false if verification fails. +// +// DecodeSlice reads the tree data 64 bytes at a time, so if the reader is +// unbuffered (e.g. os.File), wrapping it in a bufio.Reader will significantly +// improve performance. func DecodeSlice(dst io.Writer, data io.Reader, group int, offset, length uint64, root [32]byte) (bool, error) { groupSize := uint64(guts.ChunkSize << group) buf := make([]byte, groupSize) @@ -292,6 +308,13 @@ func DecodeSlice(dst io.Writer, data io.Reader, group int, offset, length uint64 if err != nil { return false } else if bufLen <= groupSize { + if bufLen == 0 { + // the tree for empty data is a single empty group; there is + // no data to decode, but we can still verify the root + n := compressGroup(nil, 0) + n.Flags |= flags + return cv == guts.ChainingValue(n) + } if !inSlice { return true } @@ -321,7 +344,7 @@ func DecodeSlice(dst io.Writer, data io.Reader, group int, offset, length uint64 } dataLen := binary.LittleEndian.Uint64(read(8)) - if dataLen < offset+length { + if end := offset + length; end < offset || dataLen < end { return false, errors.New("invalid slice length") } ok := rec(bytesToCV(root[:]), 0, dataLen, guts.FlagRoot) @@ -339,13 +362,16 @@ func VerifySlice(data []byte, group int, offset uint64, length uint64, root [32] return buf.Bytes(), true } -// VerifyChunks verifies the provided chunks with a full outboard encoding. +// VerifyChunk verifies the provided chunks with a full outboard encoding. func VerifyChunk(chunks, outboard []byte, group int, offset uint64, root [32]byte) bool { cbuf := bytes.NewBuffer(chunks) obuf := bytes.NewBuffer(outboard) groupSize := uint64(guts.ChunkSize << group) length := uint64(len(chunks)) nodesWithin := func(bufLen uint64) int { + if bufLen <= groupSize { + return 0 // leaf + } n := int(bufLen / groupSize) if bufLen%groupSize == 0 { n-- @@ -357,6 +383,13 @@ func VerifyChunk(chunks, outboard []byte, group int, offset uint64, root [32]byt rec = func(cv [8]uint32, pos, bufLen uint64, flags uint32) bool { inSlice := pos < (offset+length) && offset < (pos+bufLen) if bufLen <= groupSize { + if bufLen == 0 { + // the tree for empty data is a single empty group; there are + // no chunks to verify, but we can still verify the root + n := compressGroup(nil, 0) + n.Flags |= flags + return cv == guts.ChainingValue(n) + } if !inSlice { return true } @@ -378,7 +411,7 @@ func VerifyChunk(chunks, outboard []byte, group int, offset uint64, root [32]byt return false } dataLen := binary.LittleEndian.Uint64(obuf.Next(8)) - if dataLen < offset+length || obuf.Len() != 64*nodesWithin(dataLen) { + if end := offset + length; end < offset || dataLen < end || obuf.Len() != 64*nodesWithin(dataLen) { return false } return rec(bytesToCV(root[:]), 0, dataLen, guts.FlagRoot) diff --git a/bao/bao_test.go b/bao/bao_test.go index 7c91533..f42ab26 100644 --- a/bao/bao_test.go +++ b/bao/bao_test.go @@ -4,12 +4,13 @@ import ( "bytes" "encoding/binary" "encoding/hex" - "fmt" + "io" + "math" "os" "testing" - "lukechampine.com/blake3" - "lukechampine.com/blake3/bao" + "github.com/forkcloser/blake3" + "github.com/forkcloser/blake3/bao" ) func toHex(data []byte) string { return hex.EncodeToString(data) } @@ -44,19 +45,21 @@ func TestBaoGolden(t *testing.T) { // test empty input interleaved, root = bao.EncodeBuf(nil, 0, false) - if toHex(root[:]) != "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" { + switch { + case toHex(root[:]) != "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262": t.Errorf("bad root: %x", root) - } else if toHex(interleaved[:]) != "0000000000000000" { + case toHex(interleaved) != "0000000000000000": t.Errorf("bad interleaved encoding: %x", interleaved) - } else if !bao.VerifyBuf(interleaved, nil, 0, root) { + case !bao.VerifyBuf(interleaved, nil, 0, root): t.Error("verify failed") } outboard, root = bao.EncodeBuf(nil, 0, true) - if toHex(root[:]) != "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" { + switch { + case toHex(root[:]) != "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262": t.Errorf("bad root: %x", root) - } else if toHex(outboard[:]) != "0000000000000000" { + case toHex(outboard) != "0000000000000000": t.Errorf("bad outboard encoding: %x", outboard) - } else if !bao.VerifyBuf(nil, outboard, 0, root) { + case !bao.VerifyBuf(nil, outboard, 0, root): t.Error("verify failed") } } @@ -65,7 +68,7 @@ func TestBaoInterleaved(t *testing.T) { data := make([]byte, 1<<20) blake3.New(0, nil).XOF().Read(data) - for group := 0; group < 10; group++ { + for group := range 10 { interleaved, root := bao.EncodeBuf(data, group, false) if !bao.VerifyBuf(interleaved, nil, group, root) { t.Fatal("verify failed") @@ -101,7 +104,7 @@ func TestBaoOutboard(t *testing.T) { data := make([]byte, 1<<20) blake3.New(0, nil).XOF().Read(data) - for group := 0; group < 10; group++ { + for group := range 10 { outboard, root := bao.EncodeBuf(data, group, true) if !bao.VerifyBuf(data, outboard, group, root) { t.Fatal("verify failed") @@ -152,12 +155,75 @@ func TestBaoChunkGroup(t *testing.T) { } { input := baoInput(test.inputLen) _, root := bao.EncodeBuf(input, group, false) - if out := fmt.Sprintf("%x", root); out != test.exp { + if out := toHex(root[:]); out != test.exp { t.Errorf("output %v did not match test vector:\n\texpected: %v...\n\t got: %v...", test.inputLen, test.exp[:10], out[:10]) } } } +func TestBaoVerifyChunk(t *testing.T) { + data := make([]byte, 1<<18) + blake3.New(0, nil).XOF().Read(data) + + for _, group := range []int{0, 4} { + groupSize := 1024 << group + outboard, root := bao.EncodeBuf(data, group, true) + for _, g := range []int{0, 1, 5, len(data)/groupSize - 1} { + off := g * groupSize + chunk := data[off:][:groupSize] + if !bao.VerifyChunk(chunk, outboard, group, uint64(off), root) { + t.Errorf("group %v: verify failed at offset %v", group, off) + } + badChunk := append([]byte(nil), chunk...) + badChunk[0] ^= 1 + if bao.VerifyChunk(badChunk, outboard, group, uint64(off), root) { + t.Errorf("group %v: verify succeeded with corrupted chunk at offset %v", group, off) + } + } + // multiple contiguous groups + if !bao.VerifyChunk(data[:4*groupSize], outboard, group, 0, root) { + t.Errorf("group %v: verify failed for contiguous groups", group) + } + } + + // the encoding of empty data has no chunks, but its root should still be + // verified + outboard, root := bao.EncodeBuf(nil, 0, true) + if !bao.VerifyChunk(nil, outboard, 0, 0, root) { + t.Error("verify failed for empty encoding") + } + badRoot := root + badRoot[0] ^= 1 + if bao.VerifyChunk(nil, outboard, 0, 0, badRoot) { + t.Error("verify succeeded for empty encoding with bad root") + } +} + +func TestBaoInvalidSliceBounds(t *testing.T) { + data := make([]byte, 4096) + blake3.New(0, nil).XOF().Read(data) + enc, root := bao.EncodeBuf(data, 0, false) + + for _, test := range []struct { + off, len uint64 + }{ + {0, 4097}, // out of range + {4096, 1}, // out of range + {1 << 63, 1<<63 + 10}, // offset+length overflows + {math.MaxUint64, math.MaxUint64}, // offset+length overflows + } { + if err := bao.ExtractSlice(io.Discard, bytes.NewReader(enc), nil, 0, test.off, test.len); err == nil { + t.Errorf("ExtractSlice accepted invalid slice bounds (%v, %v)", test.off, test.len) + } + if ok, err := bao.DecodeSlice(io.Discard, bytes.NewReader(enc), 0, test.off, test.len, root); ok || err == nil { + t.Errorf("DecodeSlice accepted invalid slice bounds (%v, %v)", test.off, test.len) + } + if _, ok := bao.VerifySlice(enc, 0, test.off, test.len, root); ok { + t.Errorf("VerifySlice accepted invalid slice bounds (%v, %v)", test.off, test.len) + } + } +} + func TestBaoStreaming(t *testing.T) { data := make([]byte, 1<<20) blake3.New(0, nil).XOF().Read(data) diff --git a/blake3.go b/blake3.go index d636e6e..f72670e 100644 --- a/blake3.go +++ b/blake3.go @@ -1,8 +1,7 @@ // Package blake3 implements the BLAKE3 cryptographic hash function. -package blake3 // import "lukechampine.com/blake3" +package blake3 // import "github.com/forkcloser/blake3" import ( - "bytes" "encoding/binary" "errors" "hash" @@ -12,8 +11,8 @@ import ( "runtime" "sync" - "lukechampine.com/blake3/bao" - "lukechampine.com/blake3/guts" + "github.com/forkcloser/blake3/bao" + "github.com/forkcloser/blake3/guts" ) // Hasher implements hash.Hash. @@ -80,17 +79,24 @@ func (h *Hasher) Write(p []byte) (int, error) { if rem == 0 { rem = len(h.buf) // don't prematurely compress } - eigenbuf := bytes.NewBuffer(p[:len(p)-rem]) - trees := guts.Eigentrees(h.counter, uint64(eigenbuf.Len()/guts.ChunkSize)) + eigenbuf := p[:len(p)-rem] + trees := guts.Eigentrees(h.counter, uint64(len(eigenbuf)/guts.ChunkSize)) cvs := make([][8]uint32, len(trees)) counter := h.counter var wg sync.WaitGroup for i, height := range trees { - wg.Add(1) - go func(i int, buf []byte, counter uint64) { - defer wg.Done() + buf := eigenbuf[:(1< 0 { - or.n.Counter = or.off / guts.BlockSize - if numBufs := len(p) / len(or.buf); numBufs < 1 { - guts.CompressBlocks(&or.buf, or.n) - n := copy(p, or.buf[or.off%bufsize:]) + // drain buffered output + if or.off >= or.bufStart && or.off-or.bufStart < uint64(or.buflen) { + n := copy(p, or.buf[or.off-or.bufStart:or.buflen]) p = p[n:] or.off += uint64(n) - } else if numBufs == 1 { - guts.CompressBlocks((*[bufsize]byte)(p), or.n) - p = p[bufsize:] - or.off += bufsize - } else { - // parallelize - par := min(numBufs, runtime.NumCPU()) - per := uint64(numBufs / par) + continue + } + if head := int(or.off % guts.BlockSize); head != 0 || len(p) < bufsize { + // the read is small or unaligned; compress (only) as many blocks + // as necessary into our buffer, and serve it from there + or.bufStart = or.off - uint64(head) + or.n.Counter = or.bufStart / guts.BlockSize + need := min(head+len(p), bufsize) + numBlocks := (need + guts.BlockSize - 1) / guts.BlockSize + or.buflen = guts.BlockSize * guts.CompressBlocksN(&or.buf, or.n, numBlocks) + continue + } + // the read is large and block-aligned; compress directly into p + or.n.Counter = or.off / guts.BlockSize + numBufs := len(p) / bufsize + const minBufsPerCPU = (16 * 1024) / bufsize + if par := min(numBufs/minBufsPerCPU, runtime.NumCPU()); par > 1 { + // enough work for each CPU to be worth parallelizing; distribute + // the buffers evenly among the goroutines var wg sync.WaitGroup - for range par { + for i := range par { + bufs := uint64(numBufs / par) + if i < numBufs%par { + bufs++ + } wg.Add(1) - go func(p []byte, n guts.Node) { + go func(p []byte, n guts.Node, bufs uint64) { defer wg.Done() - for i := range per { + for i := range bufs { guts.CompressBlocks((*[bufsize]byte)(p[i*bufsize:]), n) n.Counter += bufsize / guts.BlockSize } - }(p, or.n) - p = p[per*bufsize:] - or.off += per * bufsize + }(p, or.n, bufs) + p = p[bufs*bufsize:] + or.off += bufs * bufsize or.n.Counter = or.off / guts.BlockSize } wg.Wait() + } else { + guts.CompressBlocks((*[bufsize]byte)(p), or.n) + p = p[bufsize:] + or.off += bufsize } } return lenp, nil @@ -306,19 +334,22 @@ func (or *OutputReader) Seek(offset int64, whence int) (int64, error) { return 0, errors.New("seek position cannot be negative") } off -= uint64(-offset) - } else { - off += uint64(offset) + } else if off += uint64(offset); off < uint64(offset) { + return 0, errors.New("seek position cannot exceed end of stream") } case io.SeekEnd: + if offset > 0 { + return 0, errors.New("seek position cannot exceed end of stream") + } off = uint64(offset) - 1 default: panic("invalid whence") } or.off = off - or.n.Counter = uint64(off) / guts.BlockSize - if or.off%(guts.MaxSIMD*guts.BlockSize) != 0 { - guts.CompressBlocks(&or.buf, or.n) - } + // NOTE: there is no need to update or invalidate the buffer: it caches an + // absolute range [bufStart, bufStart+buflen) of the stream, and Read only + // serves from it when or.off falls within that range. + // // NOTE: or.off >= 2^63 will result in a negative return value. // Nothing we can do about this. return int64(or.off), nil @@ -327,7 +358,7 @@ func (or *OutputReader) Seek(offset int64, whence int) (int64, error) { // ensure that Hasher implements hash.Hash var _ hash.Hash = (*Hasher)(nil) -// EncodedSize returns the size of a Bao encoding for the provided quantity +// BaoEncodedSize returns the size of a Bao encoding for the provided quantity // of data. // // Deprecated: Use bao.EncodedSize instead. diff --git a/blake3_test.go b/blake3_test.go index e48677c..def5d2b 100644 --- a/blake3_test.go +++ b/blake3_test.go @@ -4,13 +4,16 @@ import ( "bytes" "encoding/hex" "encoding/json" - "fmt" + "errors" "io" + "math" + "math/rand" "os" + "strconv" "testing" - "lukechampine.com/blake3" - "lukechampine.com/blake3/guts" + "github.com/forkcloser/blake3" + "github.com/forkcloser/blake3/guts" ) func toHex(data []byte) string { return hex.EncodeToString(data) } @@ -74,12 +77,14 @@ func TestXOF(t *testing.T) { for _, vec := range testVectors.Cases { in := testInput[:vec.InputLen] - // XOF should produce same output as Sum, even when outputting 7 bytes at a time + // XOF should produce same output as Sum, even when outputting 7 bytes at a time. + // Read well past the digest length, so that the seek tests below stay + // within the reference buffer. h := blake3.New(len(vec.Hash)/2, nil) h.Write(in) var xofBuf bytes.Buffer - io.CopyBuffer(&xofBuf, io.LimitReader(h.XOF(), int64(len(vec.Hash)/2)), make([]byte, 7)) - if out := toHex(xofBuf.Bytes()); out != vec.Hash { + io.CopyBuffer(&xofBuf, io.LimitReader(h.XOF(), 4096), make([]byte, 7)) + if out := toHex(xofBuf.Bytes()[:len(vec.Hash)/2]); out != vec.Hash { t.Errorf("XOF output did not match test vector:\n\texpected: %v...\n\t got: %v...", vec.Hash[:10], out[:10]) } @@ -135,7 +140,7 @@ func TestXOF(t *testing.T) { t.Errorf("expected (1000, nil) when reading near end of stream, got (%v, %v)", n, err) } n, err = xof.Read(buf) - if n != 0 || err != io.EOF { + if n != 0 || !errors.Is(err, io.EOF) { t.Errorf("expected (0, EOF) when reading past end of stream, got (%v, %v)", n, err) } @@ -149,6 +154,15 @@ func TestXOF(t *testing.T) { if err == nil { t.Error("expected invalid offset error, got nil") } + _, err = xof.Seek(1, io.SeekEnd) + if err == nil { + t.Error("expected past-end error, got nil") + } + xof.Seek(-10, io.SeekEnd) + _, err = xof.Seek(math.MaxInt64, io.SeekCurrent) + if err == nil { + t.Error("expected past-end error, got nil") + } // test invalid seek whence didPanic := func() (p bool) { @@ -161,6 +175,123 @@ func TestXOF(t *testing.T) { } } +func TestXOFSeek(t *testing.T) { + // generate golden output, one block at a time + golden := make([]byte, 1<<16) + n := guts.CompressChunk(nil, &guts.IV, 0, 0) + n.Flags |= guts.FlagRoot + for i := 0; i < len(golden); i += guts.BlockSize { + block := guts.WordsToBytes(guts.CompressNode(n)) + copy(golden[i:], block[:]) + n.Counter++ + } + + // seeking to any offset should produce the same output as the golden + // stream, in particular offsets that are not aligned to the XOF's internal + // buffer + xof := blake3.New(0, nil).XOF() + buf := make([]byte, 100) + for _, off := range []int{0, 1, 63, 64, 65, 100, 131, 1000, 1023, 1024, 1025, 1100, 2047, 2048, 3000, len(golden) - len(buf)} { + if _, err := xof.Seek(int64(off), io.SeekStart); err != nil { + t.Fatal(err) + } else if _, err := io.ReadFull(xof, buf); err != nil { + t.Fatal(err) + } + if exp := golden[off:][:len(buf)]; !bytes.Equal(buf, exp) { + t.Errorf("Seek(%v, io.SeekStart): expected %x..., got %x...", off, exp[:8], buf[:8]) + } + } + xof.Seek(0, io.SeekStart) + io.ReadFull(xof, buf) // off = 100 + xof.Seek(100, io.SeekCurrent) + io.ReadFull(xof, buf) // off = 300 + if exp := golden[200:][:len(buf)]; !bytes.Equal(buf, exp) { + t.Errorf("Seek(100, io.SeekCurrent): expected %x..., got %x...", exp[:8], buf[:8]) + } + + // seek near the end of the stream, to a buffer-unaligned offset; this also + // exercises block counters beyond 2^32 + const rem = 1500 + off := uint64(math.MaxUint64) - rem // stream ends at 2^64 - 1 + n = guts.CompressChunk(nil, &guts.IV, 0, 0) + n.Flags |= guts.FlagRoot + n.Counter = off / guts.BlockSize + var endGolden []byte + for len(endGolden) < rem+guts.BlockSize { + block := guts.WordsToBytes(guts.CompressNode(n)) + endGolden = append(endGolden, block[:]...) + n.Counter++ + } + endGolden = endGolden[off%guts.BlockSize:][:rem] + xof.Seek(-rem, io.SeekEnd) + end := make([]byte, rem) + if _, err := io.ReadFull(xof, end); err != nil { + t.Fatal(err) + } else if !bytes.Equal(end, endGolden) { + t.Errorf("Seek(-%v, io.SeekEnd): expected %x..., got %x...", rem, endGolden[:8], end[:8]) + } +} + +func TestXOFReadPatterns(t *testing.T) { + // generate golden output, one block at a time + golden := make([]byte, 1<<20) + n := guts.CompressChunk(nil, &guts.IV, 0, 0) + n.Flags |= guts.FlagRoot + for i := 0; i < len(golden); i += guts.BlockSize { + block := guts.WordsToBytes(guts.CompressNode(n)) + copy(golden[i:], block[:]) + n.Counter++ + } + + // interleave reads of various sizes (crossing the buffered, direct, and + // parallel paths) with seeks, and confirm that the output always matches + // the golden stream + rng := rand.New(rand.NewSource(0)) + xof := blake3.New(0, nil).XOF() + off := 0 + for range 500 { + if rng.Intn(4) == 0 { + off = rng.Intn(len(golden) / 2) + xof.Seek(int64(off), io.SeekStart) + } + var readSize int + switch rng.Intn(4) { + case 0: + readSize = 1 + rng.Intn(64) + case 1: + readSize = 1 + rng.Intn(2048) + case 2: + readSize = 1 + rng.Intn(1<<15) + case 3: + readSize = 1 + rng.Intn(1<<19) + } + readSize = min(readSize, len(golden)-off) + buf := make([]byte, readSize) + if _, err := io.ReadFull(xof, buf); err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf, golden[off:][:readSize]) { + t.Fatalf("read of %v bytes at offset %v did not match golden output", readSize, off) + } + off += readSize + } +} + +func TestNewValidation(t *testing.T) { + expectPanic := func(desc string, fn func()) { + t.Helper() + defer func() { + if recover() == nil { + t.Errorf("expected panic from %v", desc) + } + }() + fn() + } + expectPanic("negative size", func() { blake3.New(-1, nil) }) + expectPanic("short key", func() { blake3.New(32, make([]byte, 16)) }) + expectPanic("long key", func() { blake3.New(32, make([]byte, 33)) }) +} + func TestSum(t *testing.T) { for _, vec := range testVectors.Cases { in := testInput[:vec.InputLen] @@ -205,8 +336,8 @@ func TestReset(t *testing.T) { } func TestEigentrees(t *testing.T) { - for i := uint64(0); i < 64; i++ { - for j := uint64(0); j < 64; j++ { + for i := range uint64(64) { + for j := range uint64(64) { trees := guts.Eigentrees(i, j) x := i for _, tree := range trees { @@ -247,12 +378,12 @@ func BenchmarkWrite(b *testing.B) { func BenchmarkXOF(b *testing.B) { for _, size := range []int64{64, 1024, 65536, 1048576} { - b.Run(fmt.Sprint(size), func(b *testing.B) { + b.Run(strconv.FormatInt(size, 10), func(b *testing.B) { b.ReportAllocs() b.SetBytes(size) buf := make([]byte, size) xof := blake3.New(0, nil).XOF() - for i := 0; i < b.N; i++ { + for b.Loop() { xof.Seek(0, 0) xof.Read(buf) } @@ -262,11 +393,11 @@ func BenchmarkXOF(b *testing.B) { func BenchmarkSum256(b *testing.B) { for _, size := range []int64{64, 1024, 65536, 1048576} { - b.Run(fmt.Sprint(size), func(b *testing.B) { + b.Run(strconv.FormatInt(size, 10), func(b *testing.B) { b.ReportAllocs() b.SetBytes(size) buf := make([]byte, size) - for i := 0; i < b.N; i++ { + for b.Loop() { blake3.Sum256(buf) } }) diff --git a/go.mod b/go.mod index 144b172..c6840f7 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ -module lukechampine.com/blake3 +module github.com/forkcloser/blake3 -go 1.22 +go 1.25 -require github.com/klauspost/cpuid/v2 v2.0.9 +require github.com/klauspost/cpuid/v2 v2.4.0 -retract v1.4.0 // https://github.com/lukechampine/blake3/pull/26 +require golang.org/x/sys v0.41.0 // indirect diff --git a/go.sum b/go.sum index a389a66..a98599c 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= diff --git a/guts/compress_amd64.go b/guts/compress_amd64.go index 16377f4..d105153 100644 --- a/guts/compress_amd64.go +++ b/guts/compress_amd64.go @@ -4,8 +4,6 @@ import ( "unsafe" ) -//go:generate go run avo/gen.go -out blake3_amd64.s - //go:noescape func compressChunksAVX512(cvs *[16][8]uint32, buf *[16 * ChunkSize]byte, key *[8]uint32, counter uint64, flags uint32) @@ -108,6 +106,24 @@ func CompressBlocks(out *[MaxSIMD * BlockSize]byte, n Node) { } } +// CompressBlocksN compresses at least numBlocks copies of n with successive +// counter values, storing the results in out and returning the number of +// blocks computed, which may exceed numBlocks if doing so is cheap. +func CompressBlocksN(out *[MaxSIMD * BlockSize]byte, n Node, numBlocks int) int { + if haveAVX512 || haveAVX2 { + // all MaxSIMD blocks are computed in (at most) two SIMD dispatches, so + // there is nothing to save by computing fewer + CompressBlocks(out, n) + return MaxSIMD + } + outs := (*[MaxSIMD][64]byte)(unsafe.Pointer(out)) + for i := range numBlocks { + outs[i] = WordsToBytes(CompressNode(n)) + n.Counter++ + } + return numBlocks +} + func mergeSubtrees(cvs *[MaxSIMD][8]uint32, numCVs uint64, key *[8]uint32, flags uint32) Node { if !haveAVX2 { return mergeSubtreesGeneric(cvs, numCVs, key, flags) diff --git a/guts/compress_amd64.s b/guts/compress_amd64.s index 66c6bb1..c6095e8 100644 --- a/guts/compress_amd64.s +++ b/guts/compress_amd64.s @@ -1,4 +1,4 @@ -// Code generated by command: go run gen.go -out compress_amd64.s. DO NOT EDIT. +// Code generated by command: go run gen.go -out ../guts/compress_amd64.s. DO NOT EDIT. #include "textflag.h" diff --git a/guts/compress_generic.go b/guts/compress_generic.go index 6572836..6c78258 100644 --- a/guts/compress_generic.go +++ b/guts/compress_generic.go @@ -1,7 +1,6 @@ package guts import ( - "bytes" "math/bits" ) @@ -116,8 +115,10 @@ func compressBufferGeneric(buf *[MaxSIMD * ChunkSize]byte, buflen int, key *[8]u } var cvs [MaxSIMD][8]uint32 var numCVs uint64 - for bb := bytes.NewBuffer(buf[:buflen]); bb.Len() > 0; numCVs++ { - cvs[numCVs] = ChainingValue(CompressChunk(bb.Next(ChunkSize), key, counter+numCVs, flags)) + for i := 0; i < buflen; i += ChunkSize { + chunk := buf[i:min(i+ChunkSize, buflen)] + cvs[numCVs] = ChainingValue(CompressChunk(chunk, key, counter+numCVs, flags)) + numCVs++ } return mergeSubtrees(&cvs, numCVs, key, flags) } diff --git a/guts/compress_noasm.go b/guts/compress_noasm.go index 2db1337..1bcd2ba 100644 --- a/guts/compress_noasm.go +++ b/guts/compress_noasm.go @@ -1,5 +1,4 @@ //go:build !amd64 -// +build !amd64 package guts @@ -48,6 +47,18 @@ func CompressBlocks(out *[MaxSIMD * BlockSize]byte, n Node) { } } +// CompressBlocksN compresses at least numBlocks copies of n with successive +// counter values, storing the results in out and returning the number of +// blocks computed, which may exceed numBlocks if doing so is cheap. +func CompressBlocksN(out *[MaxSIMD * BlockSize]byte, n Node, numBlocks int) int { + for i := range numBlocks { + block := WordsToBytes(CompressNode(n)) + copy(out[i*BlockSize:], block[:]) + n.Counter++ + } + return numBlocks +} + func mergeSubtrees(cvs *[MaxSIMD][8]uint32, numCVs uint64, key *[8]uint32, flags uint32) Node { return mergeSubtreesGeneric(cvs, numCVs, key, flags) } diff --git a/guts/cpu.go b/guts/cpu.go index 34e1038..c19341b 100644 --- a/guts/cpu.go +++ b/guts/cpu.go @@ -1,5 +1,8 @@ -//go:build !darwin -// +build !darwin +// The haveAVX* flags only exist on amd64: they gate the assembly kernels, +// and declaring them on other architectures would drag the cpuid dependency +// into builds that cannot use it. + +//go:build amd64 && !darwin package guts diff --git a/guts/cpu_darwin.go b/guts/cpu_darwin.go index b1b35c7..c06b560 100644 --- a/guts/cpu_darwin.go +++ b/guts/cpu_darwin.go @@ -1,3 +1,5 @@ +//go:build amd64 + package guts import ( diff --git a/guts/generate.go b/guts/generate.go new file mode 100644 index 0000000..4be839a --- /dev/null +++ b/guts/generate.go @@ -0,0 +1,7 @@ +package guts + +// The directive lives here, in a file with no build constraints, rather than +// in compress_amd64.go: `go generate` only scans files that match the host's +// build context, so a directive in an amd64-only file would silently not run +// on other machines. +//go:generate go run -C ../avo . -out ../guts/compress_amd64.s diff --git a/guts/node.go b/guts/node.go index d6cb6d0..ffff35d 100644 --- a/guts/node.go +++ b/guts/node.go @@ -66,17 +66,21 @@ func Eigentrees(counter uint64, chunks uint64) (trees []int) { // CompressEigentree compresses a buffer of 2^n chunks in parallel, returning // their root node. func CompressEigentree(buf []byte, key *[8]uint32, counter uint64, flags uint32) Node { - if numChunks := uint64(len(buf) / ChunkSize); bits.OnesCount64(numChunks) != 1 { + numChunks := uint64(len(buf) / ChunkSize) + switch { + case bits.OnesCount64(numChunks) != 1: panic("non-power-of-two eigentree size") - } else if numChunks == 1 { + case numChunks == 1: return CompressChunk(buf, key, counter, flags) - } else if numChunks <= MaxSIMD { - buflen := len(buf) + case numChunks <= MaxSIMD: if cap(buf) < MaxSIMD*ChunkSize { - buf = append(buf, make([]byte, MaxSIMD*ChunkSize-len(buf))...) + // CompressBuffer requires a full-size buffer; copy into a + // stack-allocated one rather than growing buf on the heap + var tmp [MaxSIMD * ChunkSize]byte + return CompressBuffer(&tmp, copy(tmp[:], buf), key, counter, flags) } - return CompressBuffer((*[MaxSIMD * ChunkSize]byte)(buf[:MaxSIMD*ChunkSize]), buflen, key, counter, flags) - } else { + return CompressBuffer((*[MaxSIMD * ChunkSize]byte)(buf[:MaxSIMD*ChunkSize]), len(buf), key, counter, flags) + default: cvs := make([][8]uint32, numChunks/MaxSIMD) var wg sync.WaitGroup for i := range cvs { diff --git a/renovate.json5 b/renovate.json5 new file mode 100644 index 0000000..07f85cd --- /dev/null +++ b/renovate.json5 @@ -0,0 +1,18 @@ +{ + $schema: "https://docs.renovatebot.com/renovate-schema.json", + extends: [ + "config:recommended", + // Default preset: detects owner/repo@ver and go-module packages in aqua.yaml, + // updates the standard-registry ref, and bumps aqua_version in CI/devcontainer too. + "github>aquaproj/aqua-renovate-config#2.13.0", + ], + // Supply-chain cooldown: wait before proposing a bump. Doubles as protection + // against the race where a release tag exists but its assets aren't uploaded yet. + minimumReleaseAge: "3 days", + // `just do lint commits` enforces DCO on every PR range, bot commits included. + commitBody: "Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>", + // The update-aqua-checksum workflow pushes a fix-up commit onto Renovate's + // branches; without this, Renovate treats the branch as human-modified and + // stops rebasing it. + gitIgnoredAuthors: ["41898282+github-actions[bot]@users.noreply.github.com"], +}