diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..4a65409 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +@martikan diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 0000000..3cba60a --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,101 @@ +name: Go secure Build and Deploy +on: + workflow_dispatch: + push: + tags: + - 'v*' + +permissions: read-all + +jobs: + # ========================================== + # INIT (It generates the ldflags for build) + # ========================================== + init: + runs-on: ubuntu-latest + outputs: + commit-date: ${{ steps.ldflags.outputs.commit-date }} + commit: ${{ steps.ldflags.outputs.commit }} + version: ${{ steps.ldflags.outputs.version }} + tree-state: ${{ steps.ldflags.outputs.tree-state }} + go-version: ${{ steps.go-version.outputs.go-version }} + steps: + - id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: go-version + run: echo "go-version=$(grep '^go ' go.mod | awk '{print $2}')" >> "$GITHUB_OUTPUT" + - id: ldflags + run: | + echo "commit-date=$(git log --date=iso8601-strict -1 --pretty=%ct)" >> "$GITHUB_OUTPUT" + echo "commit=$GITHUB_SHA" >> "$GITHUB_OUTPUT" + echo "version=$(git describe --tags --always --dirty | cut -c2-)" >> "$GITHUB_OUTPUT" + echo "tree-state=$(if git diff --quiet; then echo "clean"; else echo "dirty"; fi)" >> "$GITHUB_OUTPUT" + + # ========================================== + # SECURE SLSA BUILD (Runs ONLY on Tags) + # ========================================== + slsa-build: + needs: + - init + if: startsWith(github.ref, 'refs/tags/v') + permissions: + id-token: write + contents: write + actions: read + strategy: + matrix: + os: + - linux + - darwin + arch: + - amd64 + - arm64 + uses: slsa-framework/slsa-github-generator/.github/workflows/builder_go_slsa3.yml@v2.1.0 + with: + go-version: ${{ needs.init.outputs.go-version }} + config-file: .slsa-goreleaser-${{matrix.os}}-${{matrix.arch}}.yml + evaluated-envs: "COMMIT_DATE:${{needs.init.outputs.commit-date}}, COMMIT:${{needs.init.outputs.commit}}, VERSION:${{needs.init.outputs.version}}, TREE_STATE:${{needs.init.outputs.tree-state}}" + # This uploads the secure raw binaries AND the .intoto.jsonl receipt to the GitHub Release + upload-assets: true + private-repository: true + + # ========================================== + # DOCKER DEPLOY (Runs ONLY on Tags) + # ========================================== + docker-deploy: + needs: [slsa-build] + if: startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Download Secure Linux Binary + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Only download the Linux AMD64 binary for the Docker image + gh release download ${{ github.ref_name }} --pattern "artemisctl-linux-x64" + mv artemisctl-linux-x64 artemisctl + chmod +x artemisctl + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Push Docker Image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: | + ghcr.io/martikan/artemisctl:${{ github.ref_name }} + ghcr.io/martikan/artemisctl:latest diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml new file mode 100644 index 0000000..e4bec1e --- /dev/null +++ b/.github/workflows/quality-gate.yml @@ -0,0 +1,200 @@ +name: quality-gate + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] + +permissions: read-all + +jobs: + # ========================================== + # Formatting and linting + # ========================================== + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache-dependency-path: 'go.sum' + + - name: formatting + run: | + fmt=$(gofmt -l .) + if [ -n "$fmt" ]; then + echo "$fmt" + exit 1 + fi + + - name: static-check + run: go vet ./... + + + # ========================================== + # TEST & COVERAGE + # ========================================== + test-and-coverage: + runs-on: ubuntu-latest + needs: [lint] + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache-dependency-path: 'go.sum' + + - name: Install dependencies + run: go mod download + + - name: Run Tests & Generate Coverage + run: make coverage-ci + + + - name: Generate Coverage Report + run: | + total=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//') + + echo "Total coverage: $total%" + + threshold=85 + + echo "TOTAL_COVERAGE=$total" >> $GITHUB_ENV + echo "COVERAGE_THRESHOLD=$threshold" >> $GITHUB_ENV + + awk -v total="$total" -v threshold="$threshold" 'BEGIN { + if (total < threshold) { + print "COVERAGE_FAIL=true" + } else { + print "COVERAGE_FAIL=false" + } + }' >> $GITHUB_ENV + + # Extract worst packages + awk ' + NR>1 { + file=$1 + gsub(/:.*/, "", file) + + stmts=$2 + count=$3 + + total[file]+=stmts + if (count>0) covered[file]+=stmts + } + END { + for (f in total) { + cov=100*covered[f]/total[f] + printf "%s %.2f\n", f, cov + } + }' coverage.out | sort -k2 -n | head -6 > worst_files.txt + + + # Build report + echo "## 📋 Coverage Report" >> coverage-report.md + echo "" >> coverage-report.md + echo "**Total Coverage:** **${total}%**" >> coverage-report.md + echo "" >> coverage-report.md + echo "### Worst Covered Files" >> coverage-report.md + echo "" >> coverage-report.md + + echo "| Status | File | Coverage |" >> coverage-report.md + echo "|------|------|------|" >> coverage-report.md + + while read file cov; do + + cov_int=$(printf "%.0f" $cov) + + if [ "$cov_int" -lt 70 ]; then + icon="🔴" + elif [ "$cov_int" -lt 80 ]; then + icon="🟡" + elif [ "$cov_int" -lt 90 ]; then + icon="🟢" + else + icon="⭐" + fi + + echo "| $icon | $file | **${cov}%** |" >> coverage-report.md + + done < worst_files.txt + + - name: Add Job Summary + run: cat coverage-report.md >> $GITHUB_STEP_SUMMARY + + - name: Comment/Update PR coverage report + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + + const reportPath = 'coverage-report.md'; + if (!fs.existsSync(reportPath)) { + throw new Error(`Coverage report not found at ${reportPath}`); + } + const body = fs.readFileSync(reportPath, 'utf8'); + + const prNumber = context.payload.pull_request + ? context.payload.pull_request.number + : (context.issue && context.issue.number ? context.issue.number : null); + + if (!prNumber) { + throw new Error('No issue/PR number found in the current context.'); + } + + const comments = await github.paginate( + github.rest.issues.listComments, + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + } + ); + + const existing = comments.find(c => + typeof c.body === 'string' && c.body.includes('📋 Coverage Report') + ); + + if (existing) { + core.info(`Updating comment ${existing.id}`); + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body: body, + }); + } else { + core.info('Creating new comment'); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body, + }); + } + + - name: Enforce Coverage Threshold + if: always() + run: | + echo "Coverage: $TOTAL_COVERAGE%" + echo "Threshold: $COVERAGE_THRESHOLD%" + + if [ "$COVERAGE_FAIL" = "true" ]; then + echo "Coverage below threshold!" + exit 1 + fi diff --git a/.gitignore b/.gitignore index 664699f..244e517 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,10 @@ go.work.sum # env file .env -bin/ \ No newline at end of file +bin/ + +# Subagent-driven-development scratch ledger +.superpowers/ + +# Superpowers plan/spec scaffolding (design docs live in docs/design/) +docs/superpowers/ diff --git a/.slsa-goreleaser-darwin-amd64.yml b/.slsa-goreleaser-darwin-amd64.yml new file mode 100644 index 0000000..c40a4fc --- /dev/null +++ b/.slsa-goreleaser-darwin-amd64.yml @@ -0,0 +1,12 @@ +version: 1 + +env: + - GO111MODULE=on + - CGO_ENABLED=0 +goos: darwin +goarch: amd64 +main: ./cmd/artemisctl +binary: artemisctl-darwin-x64 + +ldflags: + - "-s -w -X main.Version={{ .Env.VERSION }}" diff --git a/.slsa-goreleaser-darwin-arm64.yml b/.slsa-goreleaser-darwin-arm64.yml new file mode 100644 index 0000000..923c617 --- /dev/null +++ b/.slsa-goreleaser-darwin-arm64.yml @@ -0,0 +1,11 @@ +version: 1 + +env: + - GO111MODULE=on + - CGO_ENABLED=0 +goos: darwin +goarch: arm64 +main: ./cmd/artemisctl +binary: artemisctl-darwin-arm +ldflags: + - "-s -w -X main.Version={{ .Env.VERSION }}" diff --git a/.slsa-goreleaser-linux-amd64.yml b/.slsa-goreleaser-linux-amd64.yml new file mode 100644 index 0000000..960e9d0 --- /dev/null +++ b/.slsa-goreleaser-linux-amd64.yml @@ -0,0 +1,13 @@ +version: 1 + +env: + - GO111MODULE=on + - CGO_ENABLED=0 + - GOAMD64=v3 +goos: linux +goarch: amd64 +main: ./cmd/artemisctl +binary: artemisctl-linux-x64 + +ldflags: + - "-s -w -X main.Version={{ .Env.VERSION }}" diff --git a/.slsa-goreleaser-linux-arm64.yml b/.slsa-goreleaser-linux-arm64.yml new file mode 100644 index 0000000..d844681 --- /dev/null +++ b/.slsa-goreleaser-linux-arm64.yml @@ -0,0 +1,11 @@ +version: 1 + +env: + - GO111MODULE=on + - CGO_ENABLED=0 +goos: linux +goarch: arm64 +main: ./cmd/artemisctl +binary: artemisctl-linux-arm +ldflags: + - "-s -w -X main.Version={{ .Env.VERSION }}" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0ddab72 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +# ========================================== +# ENTERPRISE SLSA-COMPLIANT DOCKERFILE +# ========================================== +# We rely exclusively on the binary compiled by the SLSA secure runner. + +# We use Google's Distroless static image. +# It contains NO shell, NO package manager, and NO utilities. +# It only contains necessary CA certificates, timezone data, and a non-root user. +FROM gcr.io/distroless/static:nonroot + +# Standard OCI labels for enterprise container registries +LABEL org.opencontainers.image.title="artemisctl" \ + org.opencontainers.image.description="CLI tool to manage Activemq Artemis brokers" \ + org.opencontainers.image.vendor="github.com/martikan/artemisctl" + +# Ensure we operate in the root directory +WORKDIR / + +# Copy the securely compiled binary passed in by GitHub Actions. +# The --chown flag guarantees our non-root user has perfect permissions. +COPY --chown=nonroot:nonroot artemisctl /artemisctl + +# Explicitly drop privileges. +# UID 65532 is the heavily restricted 'nonroot' user built into Google's Distroless images. +USER 65532:65532 + +ENTRYPOINT ["/artemisctl"] diff --git a/Makefile b/Makefile index de4b831..3778351 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,94 @@ -APP_NAME=artemisctl -BUILD_PATH=bin +# Variables +BINARY_NAME=artemisctl +MAIN_PATH=./cmd/artemisctl/main.go +BUILD_DIR=./bin +COVERAGE_FILE=coverage.out -.PHONY: clean -clean: - rm -rf $(BUILD_PATH)/ - go mod tidy +# Linker flags to strip debug information +LDFLAGS=-ldflags="-s -w" + +.PHONY: all build clean test coverage release help it-clean fixtures + +# Name of the shared, reused integration-test broker container. +IT_BROKER=artemisctl-it-broker + +# Default target when you just type 'make' +all: clean test build + +## build: Compile the CLI for your current operating system +build: + @echo "==> Building $(BINARY_NAME)..." + @mkdir -p $(BUILD_DIR) + go build -o $(BUILD_DIR)/$(BINARY_NAME) $(MAIN_PATH) + @echo "==> Done. Binary is in $(BUILD_DIR)/$(BINARY_NAME)" + +## test: Run all tests (including our Testcontainers broker tests) +test: + @echo "==> Running linting..." + gofmt -s -w . + @echo "==> Running go vet..." + go vet ./... + @echo "==> Running tests..." + go test -v -p 1 ./internal/... -.PHONY: build -build: clean - go build -o $(BUILD_PATH)/$(APP_NAME) main.go +## coverage: Run tests with coverage for CI +coverage-ci: + @echo "==> Running tests with coverage..." + go test -covermode=atomic -race -p 1 -coverprofile=$(COVERAGE_FILE) ./internal/... + @echo "==> Done." -run: build - $(BUILD_PATH)/$(APP_NAME) +## coverage: Run tests with coverage and generate an HTML report +coverage: coverage-ci + @echo "==> Generating HTML report..." + go tool cover -html=$(COVERAGE_FILE) -o coverage.html + @echo "==> Done. Open coverage.html in your browser." + +## release: Cross-compile the CLI for Linux, macOS (Darwin), and Windows +release: clean + @echo "==> Building release binaries..." + @mkdir -p $(BUILD_DIR)/release + + GOOS=linux CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOAMD64=v3 go build $(LDFLAGS) -o $(BUILD_DIR)/release/$(BINARY_NAME)-linux-amd64 $(MAIN_PATH) + + GOOS=darwin CGO_ENABLED=0 GOARCH=arm64 go build $(LDFLAGS) -o $(BUILD_DIR)/release/$(BINARY_NAME)-darwin-arm64 $(MAIN_PATH) + + @echo "==> Release binaries are in $(BUILD_DIR)/release/" + +## clean: Remove build artifacts and coverage files +clean: + @echo "==> Cleaning up..." + @rm -rf $(BUILD_DIR) + @echo "==> Cleaned." + @echo "==> go mod tidy to clean up go.mod and go.sum..." + @go mod tidy -.PHONY: release -release: clean release-x86 release-arm +## fixtures: Regenerate internal/journal/testdata from a live 2.42 container +# The harvester is an integration test gated behind ARTEMISCTL_HARVEST=1 (so +# plain `make test` skips it). It uses a dedicated container, not the shared +# reused broker, because it stops the broker to freeze the journal. +fixtures: + @echo "==> Harvesting Artemis 2.42 data-dir fixture..." + ARTEMISCTL_HARVEST=1 go test -run TestHarvestFixture -v -p 1 -timeout 15m ./internal/journal/ + @echo "==> Done. See internal/journal/testdata/" -release-x86: - CGO_ENABLED=0 go build -ldflags "-s -w" -o $(BUILD_PATH)/$(APP_NAME)-x86 main.go +## it-clean: Remove the shared integration-test broker container +# Integration tests reuse one broker (Reuse: true) and never terminate it, and +# Ryuk cannot reap reused containers (and is disabled under rootless podman), so +# the broker lingers on purpose. Run this to force-remove it. +it-clean: + @echo "==> Removing shared integration broker $(IT_BROKER)..." + @docker rm -f $(IT_BROKER) 2>/dev/null || true + @echo "==> Done." -release-arm: - CGO_ENABLED=0 GOARCH=arm64 GOOS=darwin go build -ldflags "-s -w" -o $(BUILD_PATH)/$(APP_NAME)-arm main.go \ No newline at end of file +## help: Show this help message +help: + @echo "Usage: make " + @echo "" + @echo "Targets:" + @echo " build - Compile the CLI for your current operating system" + @echo " test - Run all tests" + @echo " coverage - Run tests with coverage and generate an HTML report" + @echo " release - Cross-compile the CLI for Linux, macOS, and Windows" + @echo " fixtures - Regenerate internal/journal/testdata from a live 2.42 container" + @echo " it-clean - Remove the shared integration-test broker container" + @echo " clean - Remove build artifacts" diff --git a/README.md b/README.md index 56477a0..ea2afda 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,471 @@ # artemisctl -Lightweight client tool to interact with ActiveMQ-Artemis broker via STOMP. +A command-line tool for managing and recovering Apache ActiveMQ Artemis +brokers. It checks broker status and health, browses queues non-destructively, +can cordon the broker to stop new messages before an export, performs an +emergency drain of every message into a local store, and later redelivers those +messages once the broker is healthy again. It can also produce messages onto a +queue for testing and load generation. -Usage: +Communicates with the broker over AMQP 1.0, using the `activemq.management` +address for management operations. -consume messages from TestUser queue: +> **Contributing?** See [`docs/CONTRIBUTING.md`](docs/CONTRIBUTING.md) for how to +> report issues, run the tests, and open a pull request. -```shell -artemisctl -u artemis -p artemis -q TestUser +## Install + +Three ways to get the CLI: build from source, download a released binary, or +run the container image. + +### Build from source + +Requires Go (version pinned in `go.mod`). Clone the repo and use `make`: + +```bash +git clone https://github.com/martikan/artemisctl.git +cd artemisctl +make build # produces bin/artemisctl +make help # list all targets (build, test, coverage, release, clean) +``` + +### Download a release binary + +Tagged releases (`v*`) publish prebuilt binaries on the GitHub Releases page, +one per OS/arch: + +| Asset | Platform | +| --- | --- | +| `artemisctl-linux-x64` | Linux amd64 (built `GOAMD64=v3`) | +| `artemisctl-linux-arm` | Linux arm64 | +| `artemisctl-darwin-x64` | macOS Intel (amd64) | +| `artemisctl-darwin-arm` | macOS Apple Silicon (arm64) | + +```bash +curl -sSLO https://github.com/martikan/artemisctl/releases/latest/download/artemisctl-linux-x64 +chmod +x artemisctl-linux-x64 +./artemisctl-linux-x64 status +``` + +Each release is produced by the [SLSA3](https://slsa.dev) secure builder and +ships a `.intoto.jsonl` provenance receipt alongside the binaries, so a download +can be cryptographically verified back to the exact source and build: + +```bash +slsa-verifier verify-artifact artemisctl-linux-x64 \ + --provenance-path artemisctl-linux-x64.intoto.jsonl \ + --source-uri github.com/martikan/artemisctl +``` + +### Container image (GHCR) + +Tagged releases also push a container image to the GitHub Container Registry, +tagged both `:latest` and the version: + +```bash +docker pull ghcr.io/martikan/artemisctl:latest + +# The entrypoint is the CLI itself — pass subcommands/flags directly. +docker run --rm ghcr.io/martikan/artemisctl:latest status --url broker.internal:61616 +``` + +The image is a Google +[distroless](https://github.com/GoogleContainerTools/distroless) `static:nonroot` +base holding only the statically linked binary — no shell, no package manager, +runs as a non-root user (UID 65532). + +## Connection + +Every command connects to a single broker. Global flags: + +| Flag | Default | Description | +| --- | --- | --- | +| `--url` | `127.0.0.1:61616` | AMQP 1.0 broker address `host:port` | +| `-u`, `--username` | `artemis` | Broker username | +| `-p`, `--password` | `artemis` | Broker password | +| `--timeout` | `30s` | Max time to establish the broker connection before failing fast | + +`--timeout` bounds only the initial connection, so an unresponsive broker +errors out instead of hanging forever. It does **not** cap the running time of a +long `export` drain or `redeliver` replay, which may legitimately exceed it. + +Prefer the `ARTEMIS_PASSWORD` environment variable over `-p` for secrets; when +set, it overrides `-p`. + +```bash +export ARTEMIS_PASSWORD='…' +artemisctl status --url broker.internal:61616 -u admin +``` + +## Commands + +### `status` + +List queues and their message counts (descending). Internal and temporary +queues (`activemq.*`, `$`-prefixed, 36-char temp-queue UUIDs) are filtered out. + +```bash +artemisctl status +``` + +### `health` + +Report disk-store usage, memory usage, and producer-blocking state, mapped to a +single verdict. The process exit code reflects the verdict, so the check is +scriptable. + +| Verdict | Condition | Exit | +| --- | --- | --- | +| `OK` | all usage < 70% | 0 | +| `DEGRADED` | any usage 70–90% | 0 | +| `CRITICAL` | any usage > 90%, or the broker is blocking producers | non-zero | + +```bash +artemisctl health +``` + +**Producer-blocking** is derived from the broker's disk-full protection: Artemis +blocks all producers once disk-store usage reaches the configured +`max-disk-usage` (read live via `broker.getMaxDiskUsage`, default 90%). This +tracks the broker's own block threshold rather than a hardcoded cutoff, so it +also fires under a custom lower `max-disk-usage`. (The dedicated +`broker.isDiskFull` operation is not available on Artemis 2.31.2.) + +### `browse` + +Non-destructively peek at the messages on a queue — nothing is consumed. + +| Flag | Default | Description | +| --- | --- | --- | +| `--queue` | *(required)* | Queue to browse | +| `--limit` | `20` | Max messages to list | +| `--offset` | `0` | Skip the first N messages | +| `--message` | | Show the full body + properties for a single message ID | + +```bash +artemisctl browse --queue orders --limit 50 +artemisctl browse --queue orders --message 21 +``` + +### `produce` + +Send messages to a queue — either generated synthetic test data or messages +read from a JSON file. Use it to seed a queue for testing, reproduce a message +copied out of the Artemis web console, or drive load. + +| Flag | Default | Description | +| --- | --- | --- | +| `--queue` | *(required)* | Target queue (authoritative — a file message's `address` is ignored) | +| `--count` | `1` | Number of generated messages (ignored with `--file`) | +| `--size` | `256` | Generated message body size in bytes (ignored with `--file`) | +| `--rate` | `0` | Max messages per second in total, across all workers (`0` = unlimited) | +| `--workers` | `1` | Parallel sender sessions (`1` = ordered, sequential) | +| `--property` | | Application property `k=v` set on every message (repeatable) | +| `--file` | | JSON file of messages to send (see below) | + +Messages are sent **durable** by default. `Ctrl-C` stops cleanly between +messages, reporting how many were sent. + +Each send waits for the broker to settle the message, so single-worker +throughput is capped at one broker round-trip per message. `--workers N` opens +N senders, each on its own AMQP session over the one connection, so those +settlement waits overlap — throughput scales near-linearly with workers +(benchmarked ~110 msgs/s at 1 worker vs ~1400 msgs/s at 16 against a local +broker). With more than one worker, delivery **order is not preserved**; +`--rate` still bounds the total rate. Separate connections per worker measured +no better than sessions, so `produce` always uses a single connection. + +```bash +# 1000 generated 512-byte messages at 100/sec, each tagged env=test +artemisctl produce --queue orders --count 1000 --size 512 --rate 100 --property env=test + +# load generation: 8 parallel senders, as fast as the broker settles +artemisctl produce --queue orders --count 100000 --workers 8 + +# replay messages authored/exported in the Artemis console JSON layout +artemisctl produce --queue orders --file messages.json ``` -consume messages using different broker instead of localhost one: +**`--file` layout** is the Artemis web-console / `listMessagesAsJSON` shape: a +JSON **array** of message objects. The body comes from `text`; `durable` +defaults to `true` and `priority` to `4` when omitted; a non-zero `expiration` +(unix millis) sets the message's absolute expiry. Each typed property bucket is +flattened into the message's application properties with its correct AMQP type, +and any `--property` flags are merged on top (overriding a file property of the +same name). `address` and `type` are ignored. -```shell -artemisctl -b internal.demo.artemis:30202 -u artemis -p artemis -q TestUser +```json +[ + { + "address": "orders", + "durable": true, + "priority": 4, + "expiration": 0, + "type": 3, + "text": "hello world", + "StringProperties": { "region": "eu" }, + "IntProperties": { "attempt": 1 } + } +] ``` ---- +### `cordon` + +Block producers across the whole broker before an export, so the message set +does not grow while you drain it. Applies an address-full `FAIL` policy to the +match-all wildcard (`#`), so new sends are rejected with +`amqp:resource-limit-exceeded` ("Address … is full"). Consumers and `export` are +unaffected — the connection stays open and draining still works. -produce plain text messages to TestUser queue: +| Flag | Default | Description | +| --- | --- | --- | +| `--state-file` | `artemisctl-cordon.json` | Where to save the pre-cordon settings for `uncordon` | +| `--yes` | `false` | Skip the "this blocks ALL producers" confirmation prompt | -```shell -artemisctl -m P -u artemis -p artemis -q TestUser \ 11:14:48 --message '{"id": 1, "firstName": "Test1", "lastName": "Test"}' \ --message '{"id": 2, "firstName": "Test2", "lastName": "Test"}' \ --message '{"id": 3, "firstName": "Test3", "lastName": "Test"}' +```bash +artemisctl cordon # prompts, then blocks producers broker-wide +artemisctl cordon --yes # no prompt (scripting) ``` -produce json text messages to TestUser queue: +- **Reversible:** the wildcard's pre-cordon address-settings are saved to the + state file so `uncordon` can restore them exactly. +- **Broker version:** requires a broker that accepts `addAddressSettings` over + AMQP management (Artemis **2.33+**). Older brokers (e.g. 2.31.x) do not expose + the operation over AMQP; `cordon` fails fast with a clear message and changes + nothing. +- **Caveat:** the first message to an otherwise-empty address can slip in as the + cordon takes hold; every subsequent send is rejected. + +### `uncordon` + +Lift a cordon, restoring the settings saved by `cordon`. -```shell -artemisctl -m P -u artemis -p artemis -q TestUser -t json \ 11:14:48 --message '{"id": 1, "firstName": "Test1", "lastName": "Test"}' \ --message '{"id": 2, "firstName": "Test2", "lastName": "Test"}' \ --message '{"id": 3, "firstName": "Test3", "lastName": "Test"}' +| Flag | Default | Description | +| --- | --- | --- | +| `--state-file` | `artemisctl-cordon.json` | Pre-cordon settings written by `cordon` | +| `--force-remove` | `false` | Remove the wildcard settings entry instead of restoring saved state | + +```bash +artemisctl uncordon # restore from the state file, then delete it +artemisctl uncordon --force-remove # no state file? clear the wildcard entry, revert to broker defaults ``` + +### `export` + +**Destructive.** Drains every message off every user queue into a local store +file. Messages are removed from the broker as they are written. Use this to +evacuate a dying broker; the store can be replayed later with `redeliver`. + +| Flag | Default | Description | +| --- | --- | --- | +| `--out` | *(required)* | Output store file | +| `--drain-timeout` | `5s` | Idle time before a queue is considered empty | +| `--batch` | `100` | Persist/ack batch size | + +```bash +artemisctl export --out broker-2026-07-09.artx +``` + +**No-loss invariant:** for each batch, records are written and `fsync`ed to the +store *before* the messages are acknowledged on the broker. A crash between the +`fsync` and the ack leaves a message on the broker that is drained again on the +next run — an at-least-once duplicate, absorbed by redelivery dedup (below). + +### `salvage` + +**Offline.** Recovers messages from a *stopped* broker's data directory +straight off disk — no broker connection at all — and writes every +recoverable message to a local `.artx` store, replayed later with +`redeliver` exactly like an `export`ed store. Use this when the broker is +dead and won't start, so `export` (which needs a live AMQP connection) isn't +an option. See [`docs/offline-recovery.md`](docs/offline-recovery.md) for +the full step-by-step runbook, including the paging and Core-protocol +caveats. + +`salvage` is the native equivalent of `artemis data exp` — its output is a +`.artx` store replayed with `redeliver`, not XML consumed by `artemis data +imp`. + +| Flag | Default | Description | +| --- | --- | --- | +| `--data` | *(required unless `--bindings` and `--journal` are both given)* | Broker data directory; sub-dirs derived: `bindings/`, `journal/`, `large-messages/`, `paging/` | +| `--bindings` | derived from `--data` | Bindings journal dir override | +| `--journal` | derived from `--data` | Message journal dir override | +| `--large-messages` | derived from `--data` | Large-messages dir override | +| `--paging` | derived from `--data` | Paging dir override | +| `--out` | *(required)* | Output store file | +| `--force` | `false` | Proceed even if the `server.lock` live-broker probe suggests a broker is still running | +| `--allow-skips` | `false` | Exit 0 even though some messages were skipped or corruption diagnostics were reported (unsupported/corrupt data) | + +```bash +artemisctl salvage --data /mnt/rescue/data-snapshot --out rescue.artx +``` + +- **No broker connection.** `salvage` ignores the global `--url`/ + `--username`/`--password`/`--timeout` connection flags entirely — it never + dials the broker, only reads the data directory files. +- **Live-broker guard:** refuses to run if `server.lock` (probed at + `/server.lock`, `/server.lock`, or `/../server.lock`) + is flock-held, i.e. a broker process is still using that exact directory — + use `export` instead, or `--force` if you're certain the lock is stale. A + copied/snapshotted data directory passes the guard without `--force` + because nothing holds the flock on the copy. +- **Core-protocol messages:** decoded and exported alongside AMQP messages + (standard, large, and paged). Because this tool is an AMQP-1.0 client, it + cannot speak the Core wire protocol on redelivery, so `redeliver` converts + each Core record to an equivalent AMQP message before sending; the broker + re-converts it to Core for any Core/JMS consumer. Conversion covers the + common body types (text/bytes/map/object/stream) and standard headers; a Core + message that cannot be converted is skipped on redelivery (left in the store), + never silently dropped. Legacy pre-persister core adds (userType 31, not + produced by modern brokers) are still reported as skips. +- **At-least-once paging:** a handful of already-consumed messages from the + most recently paged, partially-consumed page may be resurrected — matching + the tool's existing at-least-once philosophy. See the runbook for the full + caveat. +- **Empty result:** if nothing survives, the summary reports "salvaged 0 + messages" and **no output file is written** — never replay a store you + didn't get an actual path for. +- **Skips and corruption fail the exit code.** Unrecoverable records are + itemized in a `skipped:` section; damaged journal/bindings/page-file + records (bad check-size, truncated record, broken page framing, an + undecodable bindings-record body) are itemized in a separate + `diagnostics:` section, naming the file and offset. Both fail the exit + code by default, unless `--allow-skips` — a recovery tool must not + silently lose messages. Not every `diagnostics:` entry is corruption: + informational notes (missing large-messages/paging dir, orphaned + large-message files, unknown-queue fallback, benign fileID-mismatch + reuse-leftover notes) never gate the exit code. The `.artx` store is still + written even when skips or corruption diagnostics are present; only the + exit code is gated. See [`docs/offline-recovery.md`](docs/offline-recovery.md) + for the full breakdown of which diagnostics gate and which don't. + +### `redeliver` + +Replay a store file back to the broker. + +| Flag | Default | Description | +| --- | --- | --- | +| `--in` | *(required)* | Input store file | +| `--queue` | | Redirect *all* messages to this queue (default: each message's original queue) | +| `--force` | `false` | Redeliver even if broker health is `CRITICAL` | + +```bash +artemisctl redeliver --in broker-2026-07-09.artx +``` + +- **Health-gated:** refuses to run if the broker's health verdict is + `CRITICAL`, unless `--force` is passed — this prevents re-flooding a broker + that is still failing. +- **Resumable:** progress is tracked in a sidecar checkpoint file + (`.ckpt`) holding the byte offset of the last redelivered record. A + re-run resumes from there. +- **Deduplicated:** each record carries a stable id, replayed as `_AMQ_DUPL_ID`, + so re-running a completed or interrupted redelivery cannot create duplicates — + the broker drops the repeats. +- **Graceful `Ctrl-C`:** `SIGINT` cancels the replay after the in-flight record; + the last checkpoint is already durable, so the run stops cleanly and resumes + from the same point on the next invocation. `export` handles `SIGINT` the same + way — already-drained records are `fsync`ed before the interrupt returns. + +## Recovery workflow + +```bash +# 1. (Optional, Artemis 2.33+) Freeze the broker so no new messages arrive +# while you drain it. +artemisctl cordon --yes --url dying-broker:61616 + +# 2. Broker is failing — evacuate everything to a local store. +artemisctl export --out rescue.artx --url dying-broker:61616 + +# 3. Bring up a healthy broker (or repair the old one), then replay. +artemisctl health --url healthy-broker:61616 +artemisctl redeliver --in rescue.artx --url healthy-broker:61616 + +# 4. If you cordoned the old broker and it lives on, lift the block. +artemisctl uncordon --url dying-broker:61616 +``` + +If `redeliver` is interrupted, just run the same command again — it resumes from +the checkpoint and dedup prevents double-delivery. + +**Broker won't start at all?** The workflow above needs a live AMQP +connection for step 2 (`export`). If the broker is dead — process won't +start, no connection possible — recover offline from its data directory +instead: + +```bash +# 1. Snapshot the dead broker's data dir — salvage is read-only, but the +# on-disk journal is the only copy until you've salvaged it. +cp -a /var/lib/artemis/data /mnt/rescue/data-snapshot + +# 2. Salvage straight off disk — no broker connection needed. +artemisctl salvage --data /mnt/rescue/data-snapshot --out rescue.artx + +# 3. Same replay path as above, into a fresh/wiped broker. Never boot the +# old data dir again once this is done — see the runbook for why. +artemisctl health --url new-broker:61616 +artemisctl redeliver --in rescue.artx --url new-broker:61616 +``` + +See [`docs/offline-recovery.md`](docs/offline-recovery.md) for the full +runbook, including the live-broker guard, the paging/dedup caveats, and a +failure-mode appendix. + +## Store format + +The store is an **append-only write-ahead log** (WAL), chosen over a +columnar/analytics format precisely because it must survive a crash +mid-evacuation. + +``` +File header: magic "ARTX" | version (1 byte) +Record: totalLen u32 | recUUID (16 bytes) | drainedAt i64 (unix nanos) + queueLen u16 | queueName bytes + amqpLen u32 | amqp.Message.MarshalBinary() bytes + crc32 u32 +``` + +- Records hold the raw AMQP wire encoding + (`amqp.Message.MarshalBinary()`/`UnmarshalBinary()`), so redelivery is a + perfect-fidelity replay of body, properties, headers, durability, priority, + and TTL — not a reconstruction. +- `recUUID` is the deterministic per-record id (`sha256`, first 16 bytes) reused + as `_AMQ_DUPL_ID` on redelivery. It hashes a **normalized** projection of the + message — a clone with the volatile `Header.DeliveryCount`/`FirstAcquirer` and + delivery-annotations cleared — *not* the raw `rec.AMQP` bytes. Normalizing is + what makes the same broker message re-drained after a crash (its delivery-count + bumped) collapse to a single delivery; a raw hash of the wire bytes would give + the two copies different ids and defeat dedup. (Tradeoff: two content-identical + messages hash equal, so the broker drops one as a duplicate.) +- `crc32` guards each record against truncation/corruption. On a mismatch, + readers stop at that offset and report it — a truncated WAL is still readable + up to the last good record. + +## Testing + +```bash +make test # gofmt + go vet + full suite, boots real Artemis containers (Testcontainers) +make coverage # same suite with -race + coverage, writes coverage.html +``` + +Unit tests cover the `store` package (write/read round-trip, crc detection, +checkpoint seek) and message building for `produce` (generated body sizing, +Artemis-JSON parsing with typed properties). Integration tests boot a real +Artemis broker to exercise drain → redeliver round-trips, resume, dedup, the +health verdict, non-destructive browse, and `produce` → browse round-trips +(sequential and parallel-worker). Benchmarks (`go test -bench . ./internal/broker/`) +measure produce throughput at 1/2/4/8/16 workers, comparing session-level vs +connection-level parallelism. + +## Future work + +- **Parquet analytics snapshot.** A separate, **non-destructive** + `export --format parquet` snapshot path for analytics/inspection (query in + DuckDB/pandas): flattened metadata columns plus a body blob column. + Deliberately kept out of the crash-safe drain path — Parquet's write-at-close + footer makes it unreadable if a drain crashes mid-file, which is why the + emergency store is a custom append-only WAL. +- TLS/SSL connections. +- Credentials from Kubernetes secrets / Vault. +- DLQ / expiry-queue depth in `health`. diff --git a/cmd/artemisctl/main.go b/cmd/artemisctl/main.go new file mode 100644 index 0000000..5895139 --- /dev/null +++ b/cmd/artemisctl/main.go @@ -0,0 +1,26 @@ +// Command artemisctl is a command-line tool for managing and recovering Apache +// ActiveMQ Artemis brokers over AMQP 1.0: check status and health, browse +// queues non-destructively, drain a dying broker into a local store and replay +// it, and produce messages for testing. It is a thin entry point; the command +// tree lives in internal/cli and the broker operations in internal/broker. +package main + +import ( + "fmt" + "os" + + "github.com/martikan/artemisctl/internal/cli" +) + +// Version is stamped at release time via -ldflags "-X main.Version=...". +// It defaults to "dev" for local/source builds. +var Version = "dev" + +func main() { + root := cli.NewRootCmd() + root.Version = Version + if err := root.Execute(); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 0000000..c45758c --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,127 @@ +# Contributing to artemisctl + +Thanks for your interest in improving `artemisctl`. This guide covers how to +report problems, propose changes, and get a pull request merged. + +By participating you agree to keep the tone respectful and constructive — assume +good faith, keep discussions technical, and help newcomers. + +## Ways to contribute + +- **Report a bug** — open an issue with a clear reproduction. +- **Request a feature** — open an issue describing the use case before writing + code, so we can agree on scope and design first. +- **Improve docs** — fixes to the README, this guide, or the design notes under + `docs/design/` are always welcome. +- **Send a pull request** — bug fixes and features (see the workflow below). + +If you are unsure whether a change is wanted, open an issue first. A small +discussion up front is cheaper than a rejected pull request. + +## Reporting issues + +Search the existing issues first to avoid duplicates. A good bug report has: + +- **What you did** — the exact `artemisctl` command and flags. +- **What you expected** vs **what happened** — include the full error output. +- **Environment** — `artemisctl` version (or commit), OS/arch, and the Artemis + broker version you targeted. +- **A minimal reproduction** — the smallest setup that still shows the problem. + +For feature requests, describe the problem you are trying to solve, not just the +solution you have in mind — it helps us find the best fit for the tool. + +### Security issues + +Please do **not** open a public issue for a security vulnerability. Report it +privately to the maintainer (via the email on the GitHub profile, or a GitHub +security advisory) so a fix can ship before disclosure. + +## Development setup + +You need [Go](https://go.dev/dl/) (the version pinned in [`go.mod`](../go.mod), +currently 1.25) and [Docker](https://docs.docker.com/get-docker/) running — +the integration tests boot a real Artemis broker via +[Testcontainers](https://testcontainers.com/), so Docker must be available. + +```bash +git clone https://github.com/martikan/artemisctl.git +cd artemisctl + +make build # compile to bin/artemisctl +make help # list every make target +``` + +## Coding standards + +- **Format** with `gofmt` — CI rejects any unformatted file. `make test` runs + `gofmt -s -w .` for you; you can also run `gofmt -l .` to list offenders. +- **Vet** clean — `go vet ./...` must pass (also part of `make test` and CI). +- **Match the surrounding code** — naming, comment density, and idiom. Prefer + small, focused functions and explain *why* in comments, not *what*. +- **Keep changes focused** — one logical change per pull request. Unrelated + cleanups belong in their own PR. + +## Testing + +```bash +make test # gofmt + go vet + the full suite (boots Artemis containers) +make coverage # same suite with -race + coverage, writes coverage.html +``` + +- **Add tests for every change.** Bug fixes get a regression test; features get + unit and/or integration coverage. +- **Unit tests** must not need a broker — they run under `go test -short`. +- **Integration tests** may boot a container; guard them with + `if testing.Short() { t.Skip(...) }` like the existing ones so `-short` stays + broker-free. +- **Coverage gate:** CI enforces a project-wide **85%** statement coverage + threshold (see [`.github/workflows/quality-gate.yml`](../.github/workflows/quality-gate.yml)). + A pull request that drops coverage below the bar will fail. The workflow posts + a coverage report as a PR comment. + +## Commit messages + +- Use the **imperative mood** ("Add drain retry", not "Added" / "Adds"). +- Keep the subject line short (≈50 chars) and lowercase-free of a trailing dot. +- [Conventional Commits](https://www.conventionalcommits.org/) prefixes + (`feat:`, `fix:`, `docs:`, `test:`, `refactor:`, `chore:`) are encouraged — + they make history and release notes easier to read. +- Explain the **why** in the body when it isn't obvious from the diff. + +## Pull request workflow + +1. **Fork** the repo and create a topic branch off `main` + (`git checkout -b fix/browse-offset`). +2. **Make your change** with tests, keeping the branch focused. +3. **Run the gate locally** before pushing: + + ```bash + gofmt -l . # must print nothing + go vet ./... + make coverage # tests pass, coverage stays >= 85% + ``` + +4. **Open the pull request** against `main`. Describe what changed and why, and + link any related issue (`Closes #123`). +5. **Green CI is required.** The `quality-gate` workflow runs lint, the full test + suite, and the coverage check on every pull request. +6. **Address review feedback** by pushing follow-up commits. A maintainer merges + once CI is green and the review is resolved. + +Keep pull requests small where you can — they are faster to review and safer to +merge. + +## Releases + +Releases are cut by maintainers by pushing a `v*` tag. That triggers the +[`ci-cd`](../.github/workflows/ci-cd.yml) workflow, which produces +[SLSA3](https://slsa.dev) provenance-signed binaries on the GitHub Release and +pushes a container image to `ghcr.io/martikan/artemisctl`. Contributors do not +need to touch the release process. + +--- + +Questions that don't fit an issue? Open a +[discussion](https://github.com/martikan/artemisctl/discussions) or ask in the +issue tracker. Thanks for contributing! diff --git a/docs/design/2026-07-09-artemisctl-rebuild-1.0.0-design.md b/docs/design/2026-07-09-artemisctl-rebuild-1.0.0-design.md new file mode 100644 index 0000000..6b9a70f --- /dev/null +++ b/docs/design/2026-07-09-artemisctl-rebuild-1.0.0-design.md @@ -0,0 +1,182 @@ +# artemisctl 1.0.0 — Design + +**Date:** 2026-07-09 +**Branch:** `feature/rebuild-1.0.0` +**Status:** Approved + +## Purpose + +A command-line tool to manage Apache ActiveMQ Artemis brokers, focused on +operational recovery. It checks broker status/health, browses queues and +messages non-destructively, performs an emergency drain (evacuate every +message off a broker into a local store), and later redelivers those messages +back once the broker is healthy again. + +## Scope + +In scope for 1.0.0: + +- `status` — list queues and message counts +- `health` — resource usage + a redelivery-readiness verdict +- `browse` — paged, non-destructive peek with message drill-down +- `export` — destructive drain of every queue into a local binary store +- `redeliver` — replay a store back to a broker + +Explicitly out of scope for 1.0.0: + +- `produce` (test-data load generator from the reference tool) — not needed for + an ops/recovery tool. +- `consume` (destructive read to stdout/file) — superseded by `export`/`browse`. +- Multi-broker config file / named brokers — single broker per command. +- Parquet store format — see "Future work". + +## Connection model + +Single broker per command, matching the reference tool's ergonomics: + +- `--url` (default `127.0.0.1:61616`), `-u/--username`, `-p/--password` +- `ARTEMIS_PASSWORD` environment variable preferred over `-p` for secrets +- Connects over AMQP 1.0 via `github.com/Azure/go-amqp` +- Management operations go through the `activemq.management` address using the + request/reply pattern (dynamic reply-to receiver) proven in the reference. + +## Package layout + +- `cmd/artemisctl/main.go` — entrypoint, wires cobra root +- `internal/broker` — connection, management RPC, queue enumeration, drain, + browse, health probes +- `internal/store` — binary store writer/reader and checkpoint sidecar +- `internal/cli` — cobra commands, flag and environment wiring + +Rationale: the risky, novel work (drain → store → redeliver with crash-safety, +resume, and dedup) lives behind a clean `store` boundary that can be tested in +isolation from any broker. + +## Commands + +``` +artemisctl status # queues + message counts +artemisctl health # disk%, mem%, blocking + OK/DEGRADED/CRITICAL verdict +artemisctl browse --queue Q # paged summary; --limit/--offset; --message drill-down +artemisctl export --out FILE # DRAIN every queue (destructive) into a binary store +artemisctl redeliver --in FILE # replay a store to the broker; health-gated, resumable, dedup +``` + +## Binary store format (append-only WAL) + +The store is an append-only write-ahead log — chosen over a columnar/analytics +format precisely because it must survive a crash mid-evacuation. + +``` +File header: magic "ARTX" | version (1 byte) +Record: totalLen u32 | recUUID (16 bytes) | crc32 u32 + queueLen u16 | queueName bytes + drainedAt i64 (unix nanos) + amqpLen u32 | amqp.Message.MarshalBinary() bytes +``` + +- `amqp.Message.MarshalBinary()` / `UnmarshalBinary()` are public in go-amqp + v1.5.1, so records hold the raw AMQP wire encoding — perfect-fidelity body, + properties, headers, durability, priority, and TTL. Redelivery is a faithful + replay, not a reconstruction. +- `recUUID` is a deterministic per-record id, reused as `_AMQ_DUPL_ID` on + redelivery so re-running redeliver cannot create duplicates (the broker drops + repeats). +- `crc32` guards each record against truncation/corruption. On a crc mismatch, + readers stop at that offset and report it — a truncated WAL is still readable + up to the last good record. +- Redelivery progress is tracked in a sidecar checkpoint file `FILE.ckpt` + holding the byte offset of the last successfully redelivered record; resume + seeks to that offset. + +## Drain flow (`export`) + +1. Enumerate user queues via the `listQueues` management operation, filtering + out internal/temporary queues (`activemq.*`, `$`-prefixed, and 36-char + temporary-queue names — reuse the reference filter). +2. For each queue, open a consuming receiver and read until idle. "Idle" = + no message received within `--drain-timeout` (default 5s), treated as + queue-empty. +3. **Durability invariant:** for each batch, write records → `fsync` the store + → **then** acknowledge (settle/accept) the messages on the broker. Ordering + guarantees no message is lost if the process crashes mid-drain. The cost is + at-least-once: a crash between fsync and ack leaves a message on the broker + that is drained again on the next run, producing a duplicate record — which + is absorbed by redelivery dedup. + +## Redeliver flow (`redeliver`) + +1. Probe broker health. Refuse to redeliver if the verdict is `CRITICAL`, + unless `--force` is passed. Prevents re-flooding a broker that is still + failing. +2. Open the store and seek to the checkpoint offset (0 if none). +3. For each record: `UnmarshalBinary` the message, set `_AMQ_DUPL_ID` = + `recUUID`, send to the queue the message was drained from (`--queue` + overrides all messages to a single queue). On broker ack, advance and + persist the checkpoint. +4. On SIGINT, flush a clean checkpoint before exit so the next run resumes + exactly. + +## Health verdict (`health`) + +Queries management operations for disk-store usage %, address/global memory +usage %, and producer-blocking/paging state. Maps to a single verdict: + +- `OK` — all usage < 70% +- `DEGRADED` — any usage 70–90% +- `CRITICAL` — any usage > 90%, or the broker is blocking producers + +The process exit code reflects the verdict so the check is scriptable, and +`redeliver` gates on it. + +## Browse (`browse`) + +Non-destructive paged peek: + +- Default: a table of messages (id, size, timestamp, body preview) with + `--limit` / `--offset` paging. +- `--message `: full body + properties for one message. + +**Implementation spike/risk (flagged honestly):** non-destructive peek over +AMQP is less clean than management RPC. Plan: build the summary table from the +Artemis management `browse` operation (returns message metadata without moving +messages); serve full-body drill-down via a browse-mode receiver. Exact +go-amqp browse-mode support needs a short spike during implementation before +this command is finalized. + +## Error handling + +- Connection/auth failure: fail fast with a clear message and non-zero exit. +- Partial drain: already-acked messages are safely in the store; unacked + messages remain on the broker for the next run. +- Corrupt store record (crc mismatch): stop reading, report the byte offset, + redeliver only the good prefix. +- Interrupted redelivery (SIGINT): flush the checkpoint cleanly so resume is + exact. + +## Testing + +Integration tests use Testcontainers Artemis (already a dependency pattern in +the reference tool): + +- drain → redeliver round-trip preserves message bodies and properties +- resume after a killed redelivery re-sends only remaining messages +- dedup: re-running a completed redelivery produces no duplicates on the broker +- health verdict maps usage thresholds to OK/DEGRADED/CRITICAL correctly +- corrupt/truncated store is read up to the last good record + +Unit tests cover the `store` package (write/read round-trip, crc detection, +checkpoint seek) with no broker required. + +## Future work + +- **Parquet analytics snapshot.** Add `export --format parquet` as a separate, + **non-destructive** snapshot path for analytics/inspection (query in + DuckDB/pandas): flattened metadata columns plus a body blob column, footer + finalized because it is not evacuating a dying broker. Deliberately kept out + of the crash-safe drain path — Parquet's write-at-close footer makes it + unreadable if a drain crashes mid-file, which is why the emergency store is a + custom append-only WAL. +- TLS/SSL connections. +- Credentials from Kubernetes secrets / Vault. +- DLQ / expiry-queue depth in `health`. diff --git a/docs/offline-recovery.md b/docs/offline-recovery.md new file mode 100644 index 0000000..96305c0 --- /dev/null +++ b/docs/offline-recovery.md @@ -0,0 +1,229 @@ +# Offline recovery runbook + +This runbook is for the scenario `export` can't handle: the broker is **dead** +and won't start, so there's no AMQP connection to drain. Its messages are only +reachable as files on disk (data directory, possibly on an NFS mount that +survived the crash). `salvage` reads that data directory directly — no JVM, no +running broker — and produces the same `.artx` store that `export` produces, +replayed with the same `redeliver` command. + +If the broker **is** still up (even in a degraded state), don't use this +document — use the live-broker path in the main [README](../README.md#recovery-workflow) +(`cordon` + `export`). Salvage is a strictly worse recovery than a live drain: +it can resurrect a handful of already-consumed paged messages (see the caveat +in step 3). Reach for it only when `export` truly isn't an option. + +## Decision table + +| Broker state | Path | +| --- | --- | +| Alive (even degraded/CRITICAL) — accepts an AMQP connection | `cordon` (optional, freezes producers) then `export` — see the [README recovery workflow](../README.md#recovery-workflow) | +| Dead — process won't start, no AMQP connection possible | This document: `salvage` → `redeliver` | + +If you're not sure which one you're in, try to connect first: + +```bash +artemisctl status --url dead-broker:61616 +``` + +If that times out or errors and the broker process genuinely won't come back +up, you're in the dead-broker case below. + +## Procedure + +### 1. Confirm the broker is dead + +Check for a live broker process and a held `server.lock` on the data +directory before doing anything else — `salvage` refuses to run against a +data directory whose broker process is still holding that lock (see step 3), +but it's worth confirming manually too so you don't spend time on the wrong +path. If the broker process is running or can be started, stop — go use +`cordon` + `export` instead (see the decision table above). + +### 2. Snapshot the data dir + +`salvage` opens every file `O_RDONLY` and never writes into the data +directory it reads, but the on-disk journal is the *only* copy of these +messages until you've salvaged them — always work from a copy, never the +broker's live directory: + +```bash +cp -a /var/lib/artemis/data /mnt/rescue/data-snapshot-2026-07-12 +# or, on NFS: take a filesystem/volume snapshot instead of cp -a +``` + +A copied/snapshotted data dir also sidesteps the live-broker lock probe +cleanly: the `server.lock` file is copied along with everything else, but +nothing holds an flock on the *copy*, so step 3's guard passes without +needing `--force`. + +### 3. Salvage + +Run `salvage` against the snapshot, not the original: + +```bash +artemisctl salvage --data /mnt/rescue/data-snapshot-2026-07-12 \ + --out rescue-2026-07-12.artx +``` + +`--data` is the broker data directory; `salvage` derives `bindings/`, +`journal/`, `large-messages/`, and `paging/` sub-directories from it. Pass +those individually instead (`--bindings`, `--journal`, `--large-messages`, +`--paging`) if your layout doesn't match, or if you only have `--bindings` +and `--journal` on hand. + +`salvage` is **fully offline** — it never dials the broker and ignores the +global `--url`/`--username`/`--password`/`--timeout` connection flags +entirely. + +**Live-broker guard:** before touching anything, `salvage` probes for a held +`server.lock` (checking `/server.lock`, then `/server.lock`, +then `/../server.lock`, using whichever exists first) and refuses to +run if it's flock-held — that means a broker process is still using this +exact directory, and you should use `export` instead. Pass `--force` only if +you're certain the lock is stale (e.g. unreliable NFS lock semantics) and no +broker process actually holds it. + +Read the summary printed at the end: + +``` +salvaged 512 messages to rescue.artx + salvage.large 1 (largest 300.1 KiB) + salvage.paged 500 + salvage.plain 5 + salvage.props 5 + salvage.scheduled 1 +``` + +When `salvage` encounters records it cannot process, a `skipped:` section +lists each skip category (e.g., missing large-message body files, in-doubt +transactions, undecodable bodies, or legacy userType-31 core adds) with a count +of affected records. + +When `salvage` encounters damaged data -- a corrupted journal, bindings, or +page-file record (bad check-size, truncated record, broken page framing, an +undecodable bindings-record body) -- a separate `diagnostics:` section lists +each incident, naming the file and byte offset (or, for a damaged +bindings-record body, the binding's record id) and the reason. Not every +`diagnostics:` line is corruption, though: some are informational notes that +never affect the exit code (a missing `large-messages`/`paging` dir the +journal doesn't actually reference, records exported under a synthetic +`unknown-queue-` name, orphaned large-message files, or a benign +fileID-mismatch note from a normally-reused journal file) -- see the +per-category description in the failure-mode appendix below for which is +which. + +By default `salvage` exits non-zero when anything was skipped **or** when a +corruption-class diagnostic was reported, specifically so neither can be +missed in a script. If a `skipped:` section or a corruption-class +`diagnostics:` entry is present, resolve or accept it before proceeding (see +the failure-mode appendix below). Pass `--allow-skips` to accept both skips +and corruption diagnostics and exit 0 anyway once you've reviewed them -- +the flag name predates the corruption-gating behavior but covers both. + +**Caveats to understand before you proceed:** + +- **Core-protocol messages** are decoded and exported alongside AMQP messages + (standard, large, and paged). On `redeliver` they are converted to equivalent + AMQP messages and sent over AMQP (this tool has no Core wire client); the + broker re-converts them to Core for Core/JMS consumers. A Core message that + cannot be converted is skipped on redelivery (left in the store), never + silently dropped. Legacy pre-persister core adds (userType 31) are still + reported as skips. +- **At-least-once paging.** Pages are honored as complete-page markers where + decodable, but within a partially consumed page, `salvage` exports the + whole page when the cursor position isn't decodable. Net effect: a handful + of messages a consumer had *already processed* before the crash may be + resurrected and redelivered. `redeliver`'s `_AMQ_DUPL_ID` dedup only + catches duplicates *within* what was salvaged (the content hash is salted + by the destination queue) — it has no way to know a message was consumed + and acknowledged before the crash, in a delivery this store never saw. If + your consumers aren't idempotent, be aware a few messages may arrive twice + after this recovery. +- **Empty result.** If nothing survives, `salvage` prints "salvaged 0 + messages" and does **not** write an output file — don't expect a + `rescue-*.artx` to exist if the summary reports zero. + +### 4. Verify the store + +Sanity-check the salvaged counts against what you expect (queue depths from +monitoring, last-known `status` output, etc.) before touching the new broker. +There's no separate "inspect" command for a `.artx` file — the salvage +summary from step 3 is the record of what's in it until you `redeliver` it. + +### 5. Stand up a fresh broker (or wipe the old data dir) + +Replay always targets a broker other than the one you salvaged from — either +a genuinely fresh instance, or the same instance with its data directory +wiped and reinitialized. + +> **Never boot the old journal again after `redeliver`.** Once you've +> replayed the salvaged store, the old data directory and the new broker are +> two divergent copies of the same messages. Booting the old journal after +> this point reintroduces messages `redeliver` already delivered, with no +> dedup between the two — pick **one** source of truth (the new broker) and +> retire the old data directory for good (step 9). + +### 6. Health-gate the new broker + +```bash +artemisctl health --url new-broker:61616 +``` + +`redeliver` (next step) itself refuses to run against a `CRITICAL` broker +unless you pass `--force`, but checking first avoids finding that out after +you've already committed to the replay. + +### 7. Replay + +```bash +artemisctl redeliver --in rescue-2026-07-12.artx --url new-broker:61616 +``` + +Add `--queue ` to redirect every message to a single destination queue +instead of each message's original queue (e.g. if the target broker uses +different queue names). + +`redeliver` is the same command used for a live-broker `export`/`redeliver` +round-trip, so the same resume/dedup/`Ctrl-C` guarantees apply: + +- **Resumable:** progress is checkpointed (`.ckpt`); a re-run resumes + from the last durable point instead of restarting. +- **Deduplicated:** each record's `_AMQ_DUPL_ID` means re-running a + completed *or* interrupted redelivery of the same store cannot create + duplicates on the broker — this is also the answer if `redeliver` gets + interrupted partway through (see the failure-mode appendix). +- **Graceful `Ctrl-C`:** `SIGINT` cancels after the in-flight record; the + last checkpoint is already durable, so re-running the same command + resumes cleanly. +- **Scheduled messages** carry their original scheduled-delivery time + through salvage (as the `x-opt-delivery-time` annotation); the broker + honors it after redelivery, so a message scheduled for the future stays + scheduled instead of delivering immediately. A scheduled time already in + the past delivers right away, which is correct. + +### 8. Verify + +```bash +artemisctl status --url new-broker:61616 +artemisctl browse --queue orders --url new-broker:61616 --limit 20 +``` + +Compare counts against the salvage summary from step 3 (accounting for any +`--queue` redirection). + +### 9. Retire the old data dir + +Once the replay is verified, rename or archive the old data directory — +don't delete it outright in case you need to audit it later, but make sure +it can never again be started as a broker (per the warning in step 5). This +closes out the recovery: the new broker is now the sole source of truth. + +## Failure-mode appendix + +| Failure | What to do | +| --- | --- | +| `redeliver` is interrupted (crash, `Ctrl-C`, network drop) | Just run the same `redeliver` command again. It resumes from the checkpoint file, and `_AMQ_DUPL_ID` dedup means any record that was already delivered before the interruption is silently dropped by the broker rather than duplicated. | +| `salvage` reports skips (partial salvage) | Read the `skipped:` section of the summary to see which categories were affected (e.g., Core-protocol messages that this AMQP-only reader cannot decode, missing large-message body files, in-doubt transactions, or undecodable message bodies). Depending on your setup, these may or may not matter: Core-protocol clients are unsupported, in-doubt transactions need manual recovery outside this tool — but a missing large-message body file means an unconsumed message's body is gone, so that skip is a real loss worth investigating (leftover orphan `.msg` files with no journal record, by contrast, are benign lazy-deletion residue and never appear as skips). If records were exported under a synthetic `unknown-queue-` name (visible in the per-queue count lines and as a `diagnostics:` note, not the skip section), re-route them with `redeliver --queue `. Once you've reviewed the skips, either accept them (`--allow-skips`, or just ignore the non-zero exit code in a script that already checked the summary) or resolve the underlying cause and re-run `salvage`. | +| `salvage` reports a corrupt-record diagnostic (damaged journal/bindings/page data) | Read the `diagnostics:` section: a corruption-class entry names the affected file and byte offset (e.g. `message journal: .../activemq-data-1.amq (offset 273): check-size mismatch`) or, for a damaged bindings-record body, the binding's record id (`bindings journal: .../bindings (offset 0): decode queue binding id 3: ...`). This is real, visible data loss — the damaged record itself could not be recovered (its message may be missing entirely, or, for a damaged binding, its messages fall back to a synthetic `unknown-queue-` name) — and by default it exits non-zero exactly like a skip. Not everything in `diagnostics:` is corruption, though: informational notes (missing large-messages/paging dir the journal doesn't reference, orphaned large-message files, benign fileID-mismatch reuse-leftover notes) never gate the exit code — only structural damage does. Once you've reviewed the corruption diagnostics, either accept them (`--allow-skips`, which covers corruption diagnostics too) or investigate the underlying disk/copy damage and re-run `salvage` against a better copy if one exists. | +| The old broker revives mid-procedure (comes back up on its own, or someone restarts it) | **Stop.** You now have two potentially-diverging sources of truth — the broker that just came back, and whatever you've salvaged/replayed so far. Do not continue the procedure blindly. Pick one source of truth: either abandon the salvage output and use the now-live broker with the normal `cordon` + `export` path instead, or shut the revived broker down again and continue treating the salvaged snapshot as authoritative. Don't let both run concurrently against the same queues. | diff --git a/go.mod b/go.mod index db25d66..7a74e5f 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,71 @@ module github.com/martikan/artemisctl -go 1.23.2 - -require github.com/go-stomp/stomp v2.1.4+incompatible +go 1.25.0 require ( + github.com/Azure/go-amqp v1.5.1 github.com/google/uuid v1.6.0 - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + github.com/spf13/cobra v1.10.2 + github.com/testcontainers/testcontainers-go v0.41.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.2+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.10.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/shirou/gopsutil/v4 v4.26.2 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/sys v0.41.0 // indirect + google.golang.org/grpc v1.80.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f8d500e..b4951df 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,177 @@ -github.com/go-stomp/stomp v2.1.4+incompatible h1:D3SheUVDOz9RsjVWkoh/1iCOwD0qWjyeTZMUZ0EXg2Y= -github.com/go-stomp/stomp v2.1.4+incompatible/go.mod h1:VqCtqNZv1226A1/79yh+rMiFUcfY3R109np+7ke4n0c= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-amqp v1.5.1 h1:WyiPTz2C3zVvDL7RLAqwWdeoYhMtX62MZzQoP09fzsU= +github.com/Azure/go-amqp v1.5.1/go.mod h1:vZAogwdrkbyK3Mla8m/CxSc/aKdnTZ4IbPxl51Y5WZE= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= +github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.41.0 h1:mfpsD0D36YgkxGj2LrIyxuwQ9i2wCKAD+ESsYM1wais= +github.com/testcontainers/testcontainers-go v0.41.0/go.mod h1:pdFrEIfaPl24zmBjerWTTYaY0M6UHsqA1YSvsoU40MI= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 h1:inYW9ZhgqiDqh6BioM7DVHHzEGVq76Db5897WLGZ5Go= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0/go.mod h1:Izur+Wt8gClgMJqO/cZ8wdeeMryJ/xxiOVgFSSfpDTY= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8= +go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/internal/broker/browse.go b/internal/broker/browse.go new file mode 100644 index 0000000..820b649 --- /dev/null +++ b/internal/broker/browse.go @@ -0,0 +1,264 @@ +// Browse mechanism confirmed in Task 10 Step 1 (spike against +// apache/activemq-artemis:2.31.2 in a throwaway internal/broker/browse_spike_test.go, +// run and deleted after verification): +// +// - Brief's Mechanism A candidate, `queue. browse []`: DOES succeed, but its +// CompositeData[] reply comes back as raw Java Object Serialization (base64, +// starting with the "rO0AB..." stream magic) wrapped in a JSON string -- not +// plain JSON fields. Parsing it would require a Java ObjectInputStream- +// compatible decoder in Go. Rejected as impractical/fragile. +// `address. browse []` doesn't exist at all: broker replies +// `AMQ229069: no operation browse/0`. +// - Brief's Mechanism B candidate, a receiver with +// ReceiverOptions{SourceCapabilities: []string{"COPY"}}: attaches fine, but +// Artemis does not honor "COPY" as browse-only -- explicitly ACCEPTing all +// messages received over such a receiver still removed them from the queue +// (count went 3 -> 0). go-amqp v1.5.1 also doesn't expose AMQP's real +// `distribution-mode` field on the public ReceiverOptions (it exists only on +// an unexported internal frames.Source), so a spec-correct COPY link can't be +// requested through the public API at all. Rejected: not actually non-destructive. +// +// What actually works, confirmed live: +// - A PLAIN AMQP receiver (no special source capabilities) that calls Receive +// and then explicitly ReleaseMessage (instead of AcceptMessage) puts the +// message straight back on the queue: repeated probes confirmed the queue's +// message count is unchanged after receiving+releasing every message on it. +// The receiver delivers messages in the queue's real browse order and each +// *amqp.Message carries everything the summary needs (AMQP message-id, +// creation time, body), so this receive-then-release peek is the single +// source of truth for BrowseQueue/BrowseMessage. +// +// Why NOT zip against `queue. listMessagesAsJSON [""]` (an earlier design): +// - listMessagesAsJSON returns clean JSON metadata (double-encoded like +// ListQueues' reply) and never spends link credit, but two live assumptions +// it was trusted for are FALSE on a real broker: +// 1. Its array order is NOT ascending-messageID / arrival order. On a real +// ExpiryQueue it came back non-monotonic (e.g. [610,609,604,606,605,602]), +// and it does NOT match the order a plain receiver delivers in. +// 2. Peeking (receive+release) reorders the order listMessagesAsJSON then +// reports -- the same messages come back in a new, stable order. So a +// re-read-and-compare guard can never confirm a stable prefix for n>1. +// Because the broker's internal messageID is not surfaced on the received +// AMQP message (msg.Properties.MessageID is the AMQP message-id, delivery +// annotations are empty), there is no key to join listMessagesAsJSON metadata +// to a received body except position -- and position is exactly what (1)/(2) +// make unsound. So we do not zip: listMessagesAsJSON is used ONLY for a cheap +// message count (how many to peek), never for per-message identity or order. +// +// Implementation: BrowseQueue counts the queue via listMessagesAsJSON, opens a +// plain receiver bounded to just enough credit to cover [0, offset+limit), +// receiving-then-releasing each message in order, and builds one BrowsedMessage +// per peeked message (AMQP message-id, creation time, body size/preview). Every +// peeked message -- including ones before offset -- is released, so nothing is +// ever consumed. BrowseMessage peeks the same way and returns the raw +// *amqp.Message whose AMQP message-id matches the requested id. +// +// Known caveat -- browsing twice in quick succession: releasing a message +// increments its delivery count and Artemis reschedules it under a brief +// redelivery-delay. While that delay is in effect the messages are invisible +// BOTH to a fresh receiver AND to listMessagesAsJSON (getMessageCount still +// counts them, but listMessagesAsJSON does not), so a browse issued within ~1-2s +// of a previous one can report the queue as momentarily empty. It refills on its +// own once the delay elapses. This is inherent to any receive-then-release peek +// (the only non-destructive body-reading mechanism go-amqp exposes) and is not a +// data loss -- nothing is ever consumed. + +package broker + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/Azure/go-amqp" +) + +// previewMaxLen bounds how much of a message body is copied into +// BrowsedMessage.Preview for the summary table. +const previewMaxLen = 80 + +// peekTimeout bounds each individual Receive call in peekBodies; a queue +// that has fewer messages than requested must not hang forever. +const peekTimeout = 3 * time.Second + +// BrowsedMessage is a one-line summary of a queued message returned by +// BrowseQueue: its AMQP message-id, body size in bytes, enqueue timestamp +// (unix millis, from the message's creation-time), and a short printable +// preview of the body. +type BrowsedMessage struct { + ID string + Size int + Timestamp int64 + Preview string +} + +// queueMessageMeta mirrors one entry of Artemis's listMessagesAsJSON reply. +// Only the count of entries is used (see queueCount); the per-message fields +// are deliberately NOT trusted for identity or order (see the package comment). +type queueMessageMeta struct { + MessageID int64 `json:"messageID"` +} + +// queueCount returns how many messages queue currently holds, via the +// queue..listMessagesAsJSON management operation. Its reply is +// double-encoded (an outer []string of length 1 wrapping the real JSON array), +// the same shape parseQueueStatsReply handles for listQueues. Only the length +// of the array is used -- to bound how many messages BrowseQueue/BrowseMessage +// need to peek -- never its order or contents. +func (c *Client) queueCount(ctx context.Context, queue string) (int, error) { + reply, err := c.callManagement(ctx, "queue."+queue, "listMessagesAsJSON", `[""]`) + if err != nil { + return 0, err + } + s, ok := reply.Value.(string) + if !ok { + return 0, fmt.Errorf("unexpected listMessagesAsJSON reply type %T", reply.Value) + } + var outer []string + if err := json.Unmarshal([]byte(s), &outer); err != nil { + return 0, fmt.Errorf("parse outer array: %w", err) + } + if len(outer) == 0 { + return 0, nil + } + var meta []queueMessageMeta + if err := json.Unmarshal([]byte(outer[0]), &meta); err != nil { + return 0, fmt.Errorf("parse message metadata: %w", err) + } + return len(meta), nil +} + +// amqpMessageID renders a received message's AMQP message-id as a string. This +// is the stable per-message identity BrowseQueue reports and BrowseMessage +// matches on (the broker's internal messageID is not surfaced over AMQP). It +// returns "" when the message carries no message-id. +func amqpMessageID(m *amqp.Message) string { + if m.Properties == nil || m.Properties.MessageID == nil { + return "" + } + if s, ok := m.Properties.MessageID.(string); ok { + return s + } + return fmt.Sprintf("%v", m.Properties.MessageID) +} + +// browsedFromMessage builds the one-line summary for a single peeked message. +func browsedFromMessage(m *amqp.Message) BrowsedMessage { + data := m.GetData() + preview := string(data) + if len(preview) > previewMaxLen { + preview = preview[:previewMaxLen] + } + var ts int64 + if m.Properties != nil && m.Properties.CreationTime != nil { + ts = m.Properties.CreationTime.UnixMilli() + } + return BrowsedMessage{ + ID: amqpMessageID(m), + Size: len(data), + Timestamp: ts, + Preview: preview, + } +} + +// peekBodies receives and immediately releases up to n messages from queue, +// in browse order, returning each raw message it saw. Every message is +// released (never accepted), so nothing is removed from the queue. If the +// queue holds fewer than n messages, a per-Receive timeout is treated as +// "nothing left to peek" and the shorter slice is returned without error. +func (c *Client) peekBodies(ctx context.Context, queue string, n int) ([]*amqp.Message, error) { + if n <= 0 { + return nil, nil + } + // Credit: -1 puts the receiver in manual-credit mode. With the default + // auto-credit mode, go-amqp automatically re-issues credit as messages + // are settled, so releasing a message here would prompt the broker to + // immediately redeliver it (or a later one) back to this same receiver + // -- beyond the n messages we asked for, and sometimes racing our + // deferred Close badly enough to abort the whole connection (observed + // live: an unsolicited redelivery arriving just as we detach). Issuing + // exactly n credits up front and never renewing them keeps this + // receiver bounded to precisely the n messages we intend to peek. + recv, err := c.sess.NewReceiver(ctx, queue, &amqp.ReceiverOptions{Credit: -1}) + if err != nil { + return nil, fmt.Errorf("open peek receiver for %s: %w", queue, err) + } + defer recv.Close(context.Background()) + if err := recv.IssueCredit(uint32(n)); err != nil { + return nil, fmt.Errorf("issue peek credit for %s: %w", queue, err) + } + + msgs := make([]*amqp.Message, 0, n) + for i := 0; i < n; i++ { + rctx, cancel := context.WithTimeout(ctx, peekTimeout) + msg, err := recv.Receive(rctx, nil) + cancel() + if err != nil { + if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { + break // queue has fewer messages than requested + } + return msgs, fmt.Errorf("peek receive from %s: %w", queue, err) + } + if err := recv.ReleaseMessage(ctx, msg); err != nil { + return msgs, fmt.Errorf("release peeked message: %w", err) + } + msgs = append(msgs, msg) + } + return msgs, nil +} + +// BrowseQueue non-destructively lists up to limit messages starting at +// offset, in queue browse order. limit<=0 means "no limit" (return everything +// from offset onward). +func (c *Client) BrowseQueue(ctx context.Context, queue string, limit, offset int) ([]BrowsedMessage, error) { + if offset < 0 { + offset = 0 + } + count, err := c.queueCount(ctx, queue) + if err != nil { + return nil, err + } + if offset >= count { + return nil, nil + } + end := count + if limit > 0 && offset+limit < end { + end = offset + limit + } + + // Peek from the head of the queue up to end; AMQP gives no way to start + // receiving mid-queue, so we must walk (and release) everything before + // offset too. Each returned message is its own source of truth -- no zip + // against listMessagesAsJSON -- so order and identity are always consistent. + bodies, err := c.peekBodies(ctx, queue, end) + if err != nil { + return nil, err + } + + out := make([]BrowsedMessage, 0, end-offset) + for i := offset; i < len(bodies); i++ { + out = append(out, browsedFromMessage(bodies[i])) + } + return out, nil +} + +// BrowseMessage non-destructively fetches the full raw message whose AMQP +// message-id (as reported in BrowsedMessage.ID) matches id on queue. +func (c *Client) BrowseMessage(ctx context.Context, queue, id string) (*amqp.Message, error) { + count, err := c.queueCount(ctx, queue) + if err != nil { + return nil, err + } + bodies, err := c.peekBodies(ctx, queue, count) + if err != nil { + return nil, err + } + for _, m := range bodies { + if amqpMessageID(m) == id { + return m, nil + } + } + return nil, fmt.Errorf("message %s not found in %s", id, queue) +} diff --git a/internal/broker/browse_integration_test.go b/internal/broker/browse_integration_test.go new file mode 100644 index 0000000..d0700d0 --- /dev/null +++ b/internal/broker/browse_integration_test.go @@ -0,0 +1,142 @@ +package broker + +import ( + "context" + "strings" + "testing" + "time" +) + +// browseWithRetry wraps BrowseQueue with the caller-side retry the browse +// contract asks for: peeking-and-releasing a batch churns Artemis's internal +// state, and verifyMetaUnchanged can transiently report the queue "changed +// during browse" even with no concurrent consumer. That error is documented as +// retryable; large batches (see TestProduceParallelIntegration) trip it often +// enough that tests must retry rather than fail on the first transient. +func browseWithRetry(ctx context.Context, t *testing.T, c *Client, queue string, limit, offset int) []BrowsedMessage { + t.Helper() + var lastErr error + for attempt := 0; attempt < 5; attempt++ { + browsed, err := c.BrowseQueue(ctx, queue, limit, offset) + if err == nil { + return browsed + } + if !strings.Contains(err.Error(), "changed during browse") { + t.Fatalf("browse: %v", err) + } + lastErr = err + select { + case <-ctx.Done(): + t.Fatalf("browse retry: %v", ctx.Err()) + case <-time.After(500 * time.Millisecond): + } + } + t.Fatalf("browse still transiently changing after retries: %v", lastErr) + return nil +} + +func TestBrowseIsNonDestructive(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx) + + if err := sendTestMessages(ctx, c, "orders", []string{"m1", "m2", "m3"}); err != nil { + t.Fatalf("send test messages: %v", err) + } + + msgs, err := c.BrowseQueue(ctx, "orders", 10, 0) + if err != nil { + t.Fatalf("browse: %v", err) + } + if len(msgs) != 3 { + t.Fatalf("want 3 browsed, got %d", len(msgs)) + } + + // The zip-by-position must actually pair the right body with the right + // metadata: previews must come back in the exact order sent, and every + // message must have picked up a distinct, non-empty broker messageID. + wantPreviews := []string{"m1", "m2", "m3"} + seenIDs := make(map[string]bool, len(msgs)) + for i, m := range msgs { + if m.Preview != wantPreviews[i] { + t.Fatalf("msgs[%d].Preview = %q, want %q (zip-by-position mismatch)", i, m.Preview, wantPreviews[i]) + } + if m.ID == "" { + t.Fatalf("msgs[%d].ID is empty", i) + } + if seenIDs[m.ID] { + t.Fatalf("msgs[%d].ID %q is a duplicate", i, m.ID) + } + seenIDs[m.ID] = true + } + m2ID := msgs[1].ID + + // Non-destructive: count unchanged. + time.Sleep(500 * time.Millisecond) + stats, err := c.ListQueues(ctx) + if err != nil { + t.Fatal(err) + } + for _, s := range stats { + if s.Name == "orders" && s.MessageCount != 3 { + t.Fatalf("browse consumed messages: orders=%d want 3", s.MessageCount) + } + } + + // Paging: limit=2, offset=1 must return exactly m2,m3 and must not + // consume anything (the offset path releases everything it walks past, + // including messages before the offset). + paged, err := c.BrowseQueue(ctx, "orders", 2, 1) + if err != nil { + t.Fatalf("browse paged: %v", err) + } + if len(paged) != 2 { + t.Fatalf("want 2 paged messages, got %d", len(paged)) + } + wantPaged := []string{"m2", "m3"} + for i, m := range paged { + if m.Preview != wantPaged[i] { + t.Fatalf("paged[%d].Preview = %q, want %q", i, m.Preview, wantPaged[i]) + } + } + stats, err = c.ListQueues(ctx) + if err != nil { + t.Fatal(err) + } + for _, s := range stats { + if s.Name == "orders" && s.MessageCount != 3 { + t.Fatalf("paged browse consumed messages: orders=%d want 3", s.MessageCount) + } + } + + // Drill-down: BrowseMessage for the m2 message's ID must return its body. + full, err := c.BrowseMessage(ctx, "orders", m2ID) + if err != nil { + t.Fatalf("browse message %s: %v", m2ID, err) + } + if got := string(full.GetData()); got != "m2" { + t.Fatalf("BrowseMessage(%s) body = %q, want %q", m2ID, got, "m2") + } + + // A messageID that isn't on the queue must return a not-found error. + if _, err := c.BrowseMessage(ctx, "orders", "999999999"); err == nil { + t.Fatal("BrowseMessage for a missing ID: want error, got nil") + } + + // An offset past the end of the queue returns nothing without error. + beyond, err := c.BrowseQueue(ctx, "orders", 10, 100) + if err != nil { + t.Fatalf("browse beyond end: %v", err) + } + if len(beyond) != 0 { + t.Fatalf("offset past end: want 0 messages, got %d", len(beyond)) + } +} diff --git a/internal/broker/client.go b/internal/broker/client.go new file mode 100644 index 0000000..8e0cbe7 --- /dev/null +++ b/internal/broker/client.go @@ -0,0 +1,75 @@ +// Package broker is artemisctl's AMQP 1.0 client for Apache ActiveMQ Artemis. +// It wraps a single broker connection and exposes the operations the CLI is +// built from: querying queues and health via the activemq.management address +// (management.go, health.go), non-destructively browsing a queue (browse.go), +// draining every queue into a local store and replaying it (drain.go, +// redeliver.go), and producing messages for testing or load (produce.go). +// +// Every operation is a method on *Client, which owns one *amqp.Conn and a +// default *amqp.Session. Operations that need parallelism or isolation open +// their own additional sessions on that same connection (see produce.go). +package broker + +import ( + "context" + "fmt" + + "github.com/Azure/go-amqp" +) + +// ConnectionProps holds the coordinates needed to reach a broker. Username and +// Password may be empty for an anonymous connection. +type ConnectionProps struct { + URL string + Username string + Password string +} + +// Client is a live connection to one broker. It carries a single AMQP +// connection and a default session shared by the short read-only operations; +// callers that need concurrency open extra sessions on conn themselves. A +// Client is created with Connect and must be released with Close. +type Client struct { + conn *amqp.Conn + sess *amqp.Session +} + +// Connect dials the broker described by p and opens the default session, +// bounded by ctx. On any failure it cleans up a half-open connection and +// returns a wrapped error. The returned Client must be closed with Close. +func Connect(ctx context.Context, p ConnectionProps) (*Client, error) { + conn, err := amqp.Dial(ctx, fmt.Sprintf("amqp://%s", buildConnectionURL(p)), nil) + if err != nil { + return nil, fmt.Errorf("dial broker: %w", err) + } + sess, err := conn.NewSession(ctx, nil) + if err != nil { + _ = conn.Close() + return nil, fmt.Errorf("open session: %w", err) + } + return &Client{conn: conn, sess: sess}, nil +} + +// Session exposes the Client's default AMQP session so callers in this package +// can open their own links on it. +func (c *Client) Session() *amqp.Session { return c.sess } + +// Close tears down the default session and the underlying connection. It is +// safe to call on a partially-initialized Client and returns the connection +// close error, if any. The ctx bounds only the session close. +func (c *Client) Close(ctx context.Context) error { + if c.sess != nil { + _ = c.sess.Close(ctx) + } + if c.conn != nil { + return c.conn.Close() + } + return nil +} + +func buildConnectionURL(p ConnectionProps) string { + if p.Username != "" && p.Password != "" { + return fmt.Sprintf("%s:%s@%s", p.Username, p.Password, p.URL) + } + return p.URL +} diff --git a/internal/broker/client_test.go b/internal/broker/client_test.go new file mode 100644 index 0000000..989b735 --- /dev/null +++ b/internal/broker/client_test.go @@ -0,0 +1,22 @@ +package broker + +import "testing" + +func TestBuildConnectionURL(t *testing.T) { + cases := []struct { + name string + in ConnectionProps + want string + }{ + {"with creds", ConnectionProps{URL: "h:61616", Username: "a", Password: "b"}, "a:b@h:61616"}, + {"no creds", ConnectionProps{URL: "h:61616"}, "h:61616"}, + {"user only", ConnectionProps{URL: "h:61616", Username: "a"}, "h:61616"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := buildConnectionURL(c.in); got != c.want { + t.Fatalf("got %q want %q", got, c.want) + } + }) + } +} diff --git a/internal/broker/cordon.go b/internal/broker/cordon.go new file mode 100644 index 0000000..9edfd2f --- /dev/null +++ b/internal/broker/cordon.go @@ -0,0 +1,184 @@ +package broker + +import ( + "context" + "encoding/json" + "fmt" + "strings" +) + +// wildcardMatch is the address-settings match that covers every address on the +// broker, so a single settings entry cordons the whole broker at once. +const wildcardMatch = "#" + +// Cordon thresholds applied to the wildcard match. We use the FAIL address-full +// policy, not BLOCK: BLOCK relies on producer credit-throttling that did not +// reliably engage for AMQP producers in testing (messages kept flowing well past +// the limit), whereas FAIL deterministically rejects a send once the address is +// full, surfacing to the producer as amqp:resource-limit-exceeded ("Address ... +// is full"). maxSizeBytes must be > 0 (Artemis guards its fullness check with +// maxSize > 0) and pageSizeBytes must be < maxSizeBytes (validated). +// maxSizeBytesRejectThreshold is what the FAIL policy actually rejects on. +// +// Caveat: the first message to an otherwise-empty address is accepted (its size +// starts at 0), then every subsequent send is rejected — so at most one small +// message per address can slip in right as the cordon takes hold. +const ( + cordonMaxSizeBytes = 100 + cordonPageSizeBytes = 50 + cordonRejectThreshold = 100 +) + +// uncordonPageSizeBytes is Artemis's default page size, restored by +// UncordonRemove when it resets the wildcard match to a permissive policy. +const uncordonPageSizeBytes = 10485760 + +// ErrBrokerTooOld is returned by Cordon when the broker does not expose the +// two-argument addAddressSettings(String,String) JSON overload over the AMQP +// management address. On such brokers (e.g. 2.31.x) the setting can only be +// applied via JMX/Jolokia, not AMQP. +var ErrBrokerTooOld = fmt.Errorf("broker does not support addAddressSettings over AMQP management; " + + "it needs a version with the JSON addAddressSettings(String,String) overload (2.33+)") + +// getWildcardSettings reads the current address-settings object for the +// wildcard match and returns it as a JSON object string. Artemis returns this +// double-encoded — an outer JSON array whose single element is itself the JSON +// string of the settings object (["{...}"]) — so we unwrap one level, the same +// shape parseQueueStatsReply handles. +func (c *Client) getWildcardSettings(ctx context.Context) (string, error) { + body := fmt.Sprintf(`[%q]`, wildcardMatch) + reply, err := c.callManagement(ctx, "broker", "getAddressSettingsAsJSON", body) + if err != nil { + return "", err + } + raw, ok := reply.Value.(string) + if !ok { + return "", fmt.Errorf("unexpected getAddressSettingsAsJSON reply type %T", reply.Value) + } + var outer []string + if err := json.Unmarshal([]byte(raw), &outer); err != nil { + return "", fmt.Errorf("parse getAddressSettingsAsJSON reply: %w", err) + } + if len(outer) == 0 { + return "", fmt.Errorf("getAddressSettingsAsJSON returned an empty array") + } + return outer[0], nil +} + +// applyAddressSettings applies a settings object (JSON) to a match via the +// two-arg addAddressSettings(String,String) overload. It translates the +// "no operation addAddressSettings/2" rejection from an older broker into the +// clearer ErrBrokerTooOld. +func (c *Client) applyAddressSettings(ctx context.Context, match, settingsJSON string) error { + body, err := json.Marshal([]interface{}{match, settingsJSON}) + if err != nil { + return fmt.Errorf("marshal addAddressSettings args: %w", err) + } + if _, err := c.callManagement(ctx, "broker", "addAddressSettings", string(body)); err != nil { + return asBrokerTooOld(err) + } + return nil +} + +// ApplyAddressSettings applies a settings object (JSON) to an address match via +// the two-arg addAddressSettings(String,String) overload. It is the exported +// form of applyAddressSettings for callers outside this package (e.g. the +// journal fixture harvester, which forces paging on one address). +func (c *Client) ApplyAddressSettings(ctx context.Context, match, settingsJSON string) error { + return c.applyAddressSettings(ctx, match, settingsJSON) +} + +// asBrokerTooOld maps the broker's "no operation addAddressSettings" rejection +// (returned by brokers without the two-arg JSON overload, e.g. 2.31.x) to the +// clearer ErrBrokerTooOld, and passes any other error through unchanged. +func asBrokerTooOld(err error) error { + if err != nil && strings.Contains(err.Error(), "no operation addAddressSettings") { + return ErrBrokerTooOld + } + return err +} + +// withCordonPolicy returns the given settings object with only the address-full +// policy and the size thresholds overridden, so every other field (DLA, expiry, +// redelivery, ...) is preserved for the duration of the cordon. +func withCordonPolicy(settingsJSON string) (string, error) { + var m map[string]json.RawMessage + if err := json.Unmarshal([]byte(settingsJSON), &m); err != nil { + return "", fmt.Errorf("parse settings object: %w", err) + } + m["addressFullMessagePolicy"] = json.RawMessage(`"FAIL"`) + m["maxSizeBytes"] = json.RawMessage(fmt.Sprintf("%d", cordonMaxSizeBytes)) + m["pageSizeBytes"] = json.RawMessage(fmt.Sprintf("%d", cordonPageSizeBytes)) + m["maxSizeBytesRejectThreshold"] = json.RawMessage(fmt.Sprintf("%d", cordonRejectThreshold)) + out, err := json.Marshal(m) + if err != nil { + return "", fmt.Errorf("marshal cordon settings: %w", err) + } + return string(out), nil +} + +// withUncordonPolicy reverses exactly the fields withCordonPolicy overrides, +// resetting the address-full policy and size thresholds to permissive values +// while preserving every other field. It is how UncordonRemove lifts a cordon +// when no saved pre-cordon state is available: on Artemis 2.42 +// removeAddressSettings does not reliably clear the wildcard override, so we +// overwrite it with a permissive policy instead. +func withUncordonPolicy(settingsJSON string) (string, error) { + var m map[string]json.RawMessage + if err := json.Unmarshal([]byte(settingsJSON), &m); err != nil { + return "", fmt.Errorf("parse settings object: %w", err) + } + m["addressFullMessagePolicy"] = json.RawMessage(`"PAGE"`) + m["maxSizeBytes"] = json.RawMessage("-1") + m["pageSizeBytes"] = json.RawMessage(fmt.Sprintf("%d", uncordonPageSizeBytes)) + m["maxSizeBytesRejectThreshold"] = json.RawMessage("-1") + out, err := json.Marshal(m) + if err != nil { + return "", fmt.Errorf("marshal uncordon settings: %w", err) + } + return string(out), nil +} + +// Cordon blocks producers across the whole broker by applying an address-full +// BLOCK policy to the wildcard match. It returns the pre-cordon settings JSON so +// the caller can persist it for a later Uncordon. On a broker too old to accept +// the setting over AMQP it returns ErrBrokerTooOld and changes nothing. +func (c *Client) Cordon(ctx context.Context) (savedSettings string, err error) { + saved, err := c.getWildcardSettings(ctx) + if err != nil { + return "", err + } + blocked, err := withCordonPolicy(saved) + if err != nil { + return "", err + } + if err := c.applyAddressSettings(ctx, wildcardMatch, blocked); err != nil { + return "", err + } + return saved, nil +} + +// Uncordon restores a previously saved settings JSON to the wildcard match, +// lifting the producer block and faithfully reinstating whatever settings were +// in place before Cordon. +func (c *Client) Uncordon(ctx context.Context, savedSettings string) error { + return c.applyAddressSettings(ctx, wildcardMatch, savedSettings) +} + +// UncordonRemove lifts the cordon when no saved pre-cordon state is available. +// It cannot use removeAddressSettings: on Artemis 2.42 that reports success but +// leaves the wildcard override in place (a no-op), so the cordon never lifts. +// Instead it reads the current wildcard settings and overwrites only the +// address-full policy and size thresholds back to permissive values, undoing +// exactly what Cordon applied while preserving every other field. +func (c *Client) UncordonRemove(ctx context.Context) error { + current, err := c.getWildcardSettings(ctx) + if err != nil { + return err + } + permissive, err := withUncordonPolicy(current) + if err != nil { + return err + } + return c.applyAddressSettings(ctx, wildcardMatch, permissive) +} diff --git a/internal/broker/cordon_integration_test.go b/internal/broker/cordon_integration_test.go new file mode 100644 index 0000000..25e9277 --- /dev/null +++ b/internal/broker/cordon_integration_test.go @@ -0,0 +1,99 @@ +package broker + +import ( + "context" + "testing" + "time" +) + +func TestCordonBlocksAndUncordonRestores(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer c.Close(ctx) + + // Before cordon: a producer can send. + warmCtx, warmCancel := context.WithTimeout(ctx, 8*time.Second) + if err := sendTestMessages(warmCtx, c, "cordon.before", []string{"ok"}); err != nil { + t.Fatalf("baseline send should succeed: %v", err) + } + warmCancel() + + // Cordon. + saved, err := c.Cordon(ctx) + if err != nil { + t.Fatalf("cordon: %v", err) + } + if saved == "" { + t.Fatal("cordon returned empty saved settings") + } + + // Under cordon: producing enough to cross the block threshold fails. + blockCtx, blockCancel := context.WithTimeout(ctx, 6*time.Second) + blockErr := sendTestMessages(blockCtx, c, "cordon.blocked", + []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}) + blockCancel() + if blockErr == nil { + t.Fatal("expected producer to be blocked while cordoned") + } + + // Uncordon restores the saved settings; producing works again. + if err := c.Uncordon(ctx, saved); err != nil { + t.Fatalf("uncordon: %v", err) + } + afterCtx, afterCancel := context.WithTimeout(ctx, 8*time.Second) + defer afterCancel() + if err := sendTestMessages(afterCtx, c, "cordon.after", []string{"back"}); err != nil { + t.Fatalf("send after uncordon should succeed: %v", err) + } +} + +func TestUncordonRemoveLiftsCordon(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer c.Close(ctx) + + if _, err := c.Cordon(ctx); err != nil { + t.Fatalf("cordon: %v", err) + } + + // Confirm the cordon is actually in force before lifting it: many sends to + // one address must fail. Without this, UncordonRemove could be a no-op and + // the test below would still pass on the first-message-accepted loophole. + blockCtx, blockCancel := context.WithTimeout(ctx, 6*time.Second) + blockErr := sendTestMessages(blockCtx, c, "cordon.remove.blocked", + []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}) + blockCancel() + if blockErr == nil { + t.Fatal("expected producer to be blocked while cordoned") + } + + if err := c.UncordonRemove(ctx); err != nil { + t.Fatalf("uncordon-remove: %v", err) + } + + // After the lift the same volume of sends must all succeed. A single send + // would pass even under the FAIL policy (the first message to an empty + // address is accepted), so send many to prove the cordon is truly gone. + afterCtx, afterCancel := context.WithTimeout(ctx, 10*time.Second) + defer afterCancel() + if err := sendTestMessages(afterCtx, c, "cordon.removed", + []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}); err != nil { + t.Fatalf("send after uncordon-remove should succeed: %v", err) + } +} diff --git a/internal/broker/cordon_test.go b/internal/broker/cordon_test.go new file mode 100644 index 0000000..c629aad --- /dev/null +++ b/internal/broker/cordon_test.go @@ -0,0 +1,108 @@ +package broker + +import ( + "encoding/json" + "errors" + "testing" +) + +func TestAsBrokerTooOld(t *testing.T) { + if got := asBrokerTooOld(nil); got != nil { + t.Errorf("nil err mapped to %v, want nil", got) + } + tooOld := errors.New(`broker rejected broker.addAddressSettings: there is no operation addAddressSettings/2`) + if got := asBrokerTooOld(tooOld); !errors.Is(got, ErrBrokerTooOld) { + t.Errorf("too-old rejection mapped to %v, want ErrBrokerTooOld", got) + } + other := errors.New("connection reset") + if got := asBrokerTooOld(other); got != other { + t.Errorf("unrelated error mapped to %v, want passthrough", got) + } +} + +func TestWithCordonPolicyOverridesAndPreserves(t *testing.T) { + in := `{"DLA":"DLQ","expiryAddress":"ExpiryQueue","addressFullMessagePolicy":"PAGE","maxSizeBytes":-1,"pageSizeBytes":10485760,"maxSizeBytesRejectThreshold":-1,"maxDeliveryAttempts":10}` + out, err := withCordonPolicy(in) + if err != nil { + t.Fatalf("withCordonPolicy: %v", err) + } + var m map[string]interface{} + if err := json.Unmarshal([]byte(out), &m); err != nil { + t.Fatalf("unmarshal result: %v", err) + } + // overridden + if m["addressFullMessagePolicy"] != "FAIL" { + t.Errorf("addressFullMessagePolicy = %v, want FAIL", m["addressFullMessagePolicy"]) + } + if m["maxSizeBytes"].(float64) != cordonMaxSizeBytes { + t.Errorf("maxSizeBytes = %v, want %d", m["maxSizeBytes"], cordonMaxSizeBytes) + } + if m["pageSizeBytes"].(float64) != cordonPageSizeBytes { + t.Errorf("pageSizeBytes = %v, want %d", m["pageSizeBytes"], cordonPageSizeBytes) + } + if m["maxSizeBytesRejectThreshold"].(float64) != cordonRejectThreshold { + t.Errorf("maxSizeBytesRejectThreshold = %v, want %d", m["maxSizeBytesRejectThreshold"], cordonRejectThreshold) + } + // preserved + if m["DLA"] != "DLQ" { + t.Errorf("DLA not preserved: %v", m["DLA"]) + } + if m["expiryAddress"] != "ExpiryQueue" { + t.Errorf("expiryAddress not preserved: %v", m["expiryAddress"]) + } + if m["maxDeliveryAttempts"].(float64) != 10 { + t.Errorf("maxDeliveryAttempts not preserved: %v", m["maxDeliveryAttempts"]) + } +} + +func TestWithCordonPolicyRejectsBadJSON(t *testing.T) { + if _, err := withCordonPolicy(`not json`); err == nil { + t.Fatal("expected error for invalid settings JSON") + } +} + +func TestWithUncordonPolicyReversesCordonAndPreserves(t *testing.T) { + // Start from a cordoned settings object; uncordon must undo exactly the + // cordon fields and preserve the rest. + cordoned, err := withCordonPolicy(`{"DLA":"DLQ","expiryAddress":"ExpiryQueue","addressFullMessagePolicy":"PAGE","maxSizeBytes":-1,"pageSizeBytes":10485760,"maxSizeBytesRejectThreshold":-1,"maxDeliveryAttempts":10}`) + if err != nil { + t.Fatalf("withCordonPolicy: %v", err) + } + out, err := withUncordonPolicy(cordoned) + if err != nil { + t.Fatalf("withUncordonPolicy: %v", err) + } + var m map[string]interface{} + if err := json.Unmarshal([]byte(out), &m); err != nil { + t.Fatalf("unmarshal result: %v", err) + } + // reversed to permissive + if m["addressFullMessagePolicy"] != "PAGE" { + t.Errorf("addressFullMessagePolicy = %v, want PAGE", m["addressFullMessagePolicy"]) + } + if m["maxSizeBytes"].(float64) != -1 { + t.Errorf("maxSizeBytes = %v, want -1", m["maxSizeBytes"]) + } + if m["pageSizeBytes"].(float64) != uncordonPageSizeBytes { + t.Errorf("pageSizeBytes = %v, want %d", m["pageSizeBytes"], uncordonPageSizeBytes) + } + if m["maxSizeBytesRejectThreshold"].(float64) != -1 { + t.Errorf("maxSizeBytesRejectThreshold = %v, want -1", m["maxSizeBytesRejectThreshold"]) + } + // preserved + if m["DLA"] != "DLQ" { + t.Errorf("DLA not preserved: %v", m["DLA"]) + } + if m["expiryAddress"] != "ExpiryQueue" { + t.Errorf("expiryAddress not preserved: %v", m["expiryAddress"]) + } + if m["maxDeliveryAttempts"].(float64) != 10 { + t.Errorf("maxDeliveryAttempts not preserved: %v", m["maxDeliveryAttempts"]) + } +} + +func TestWithUncordonPolicyRejectsBadJSON(t *testing.T) { + if _, err := withUncordonPolicy(`not json`); err == nil { + t.Fatal("expected error for invalid settings JSON") + } +} diff --git a/internal/broker/coreconvert.go b/internal/broker/coreconvert.go new file mode 100644 index 0000000..04758f4 --- /dev/null +++ b/internal/broker/coreconvert.go @@ -0,0 +1,180 @@ +package broker + +import ( + "fmt" + "strings" + "time" + + "github.com/Azure/go-amqp" + "github.com/martikan/artemisctl/internal/journal" +) + +// x-opt-jms-msg-type annotation values (Artemis AMQPMessageSupport JMS_*_TYPE), +// so a JMS/Core consumer sees the right message class after redelivery. +const ( + jmsMessageType = 0 + jmsObjectMessageType = 1 + jmsMapMessageType = 2 + jmsBytesMessageType = 3 + jmsStreamMessageType = 4 + jmsTextMessageType = 5 +) + +// coreToAMQP converts a serialized Core payload (store.Record.CorePayload, +// Kind=Core) into an amqp.Message for redelivery. It mirrors Artemis's own +// CoreAmqpConverter.fromCore / CoreMessageWrapper.createAMQPSection mapping: +// the Core body becomes the AMQP body section that matches its type, and the +// core headers/properties map onto AMQP Header/Properties/ApplicationProperties. +// go-amqp performs the actual wire marshalling; the broker re-converts the +// message back to Core for any Core/JMS consumer. +// +// It returns an error only when the payload itself cannot be parsed, so the +// caller (Redeliver) can skip-not-lose a single bad record without aborting. +func coreToAMQP(payload []byte) (*amqp.Message, error) { + p, err := journal.DecodeCorePayload(payload) + if err != nil { + return nil, fmt.Errorf("decode core payload: %w", err) + } + + msg := &amqp.Message{ + Header: &amqp.MessageHeader{Durable: p.Durable, Priority: p.Priority}, + Properties: &amqp.MessageProperties{}, + } + + setCoreBody(msg, p) + + if p.Address != "" { + to := p.Address + msg.Properties.To = &to + } + if p.Timestamp > 0 { + t := time.UnixMilli(p.Timestamp) + msg.Properties.CreationTime = &t + } + if p.Expiration > 0 { + t := time.UnixMilli(p.Expiration) + msg.Properties.AbsoluteExpiryTime = &t + } + if len(p.UserID) > 0 { + msg.Properties.UserID = append([]byte(nil), p.UserID...) + } + + msg.Annotations = amqp.Annotations{"x-opt-jms-msg-type": int8(jmsTypeFor(p.Type))} + + // Application properties: carry the user-visible core properties, dropping + // Artemis-internal keys (_AMQ*/__AMQ*) and our synthetic scheduling key. + // A few internal keys carry standard AMQP message-properties (group id and + // group sequence) rather than being pure Artemis bookkeeping: promote those + // onto Properties before the strip so message grouping survives redelivery. + for k, v := range p.Properties { + if k == scheduledMsProperty { + if ms, ok := asInt64(v); ok && ms != 0 { + msg.Annotations["x-opt-delivery-time"] = ms + } + continue + } + if k == coreGroupIDProperty { + if s, ok := v.(string); ok && s != "" { + gid := s + msg.Properties.GroupID = &gid + } + continue + } + if k == coreGroupSeqProperty { + if n, ok := asInt64(v); ok && n >= 0 { + msg.Properties.GroupSequence = uint32PtrFromInt64(n) + } + continue + } + if isInternalCoreProperty(k) { + continue + } + if msg.ApplicationProperties == nil { + msg.ApplicationProperties = map[string]any{} + } + msg.ApplicationProperties[k] = v + } + + return msg, nil +} + +// scheduledMsProperty is the synthetic property emitCoreFanout uses to carry a +// journal SET_SCHEDULED_DELIVERY_TIME through the store to redelivery. +const scheduledMsProperty = "_ARTX_SCHEDULED_MS" + +// Core internal property keys that map onto standard AMQP message-properties +// rather than application-properties (Artemis Message.HDR_GROUP_ID / +// HDR_GROUP_SEQUENCE). coreToAMQP promotes these onto Properties so message +// grouping is preserved through the Core->AMQP redelivery conversion. +const ( + coreGroupIDProperty = "_AMQ_GROUP_ID" + coreGroupSeqProperty = "_AMQ_GROUP_SEQUENCE" +) + +func uint32PtrFromInt64(n int64) *uint32 { + u := uint32(n) + return &u +} + +// setCoreBody maps the Core body to the AMQP body section matching its type. +func setCoreBody(msg *amqp.Message, p *journal.CorePayload) { + switch p.Type { + case journal.CoreTypeText: + if s, ok := journal.CoreTextBody(p.Body); ok { + msg.Value = s + return + } + msg.Data = [][]byte{p.Body} // malformed text: preserve raw bytes + case journal.CoreTypeMap: + if m, ok := journal.CoreMapBody(p.Body); ok { + msg.Value = m + return + } + msg.Data = [][]byte{p.Body} + case journal.CoreTypeObject: + msg.Data = [][]byte{p.Body} + ct := "application/x-java-serialized-object" + msg.Properties.ContentType = &ct + default: + // BYTES, DEFAULT, STREAM and anything else: raw body as a Data section. + msg.Data = [][]byte{p.Body} + } +} + +func jmsTypeFor(coreType byte) int { + switch coreType { + case journal.CoreTypeText: + return jmsTextMessageType + case journal.CoreTypeBytes: + return jmsBytesMessageType + case journal.CoreTypeMap: + return jmsMapMessageType + case journal.CoreTypeObject: + return jmsObjectMessageType + case journal.CoreTypeStream: + return jmsStreamMessageType + default: + return jmsMessageType + } +} + +// isInternalCoreProperty reports whether key is an Artemis-internal core +// property that should not be re-exposed as an AMQP application-property. +func isInternalCoreProperty(key string) bool { + return strings.HasPrefix(key, "_AMQ") || strings.HasPrefix(key, "__AMQ") +} + +func asInt64(v any) (int64, bool) { + switch n := v.(type) { + case int64: + return n, true + case int32: + return int64(n), true + case int16: + return int64(n), true + case int: + return int64(n), true + default: + return 0, false + } +} diff --git a/internal/broker/coreconvert_test.go b/internal/broker/coreconvert_test.go new file mode 100644 index 0000000..042a3e1 --- /dev/null +++ b/internal/broker/coreconvert_test.go @@ -0,0 +1,139 @@ +package broker + +import ( + "encoding/binary" + "testing" + + "github.com/martikan/artemisctl/internal/journal" +) + +// coreSimpleString builds a core nullableSimpleString (NOT_NULL flag, big-endian +// int byte-length, little-endian UTF-16 pairs) — the on-wire form of a Core +// TEXT message body. +func coreSimpleString(s string) []byte { + units := []rune(s) + data := make([]byte, 0, 1+4+len(units)*2) + data = append(data, 1) // NOT_NULL + lenBuf := make([]byte, 4) + binary.BigEndian.PutUint32(lenBuf, uint32(len(units)*2)) + data = append(data, lenBuf...) + for _, r := range units { + data = append(data, byte(r), byte(r>>8)) + } + return data +} + +func TestCoreToAMQPBytesMessage(t *testing.T) { + p := &journal.CorePayload{ + MessageID: 5, + Address: "orders", + Type: journal.CoreTypeBytes, + Durable: true, + Timestamp: 1_700_000_000_000, + Priority: 4, + Properties: map[string]any{"region": "eu", "__AMQ_CID": "internal"}, + Body: []byte("hello"), + } + msg, err := coreToAMQP(p.Encode()) + if err != nil { + t.Fatalf("coreToAMQP: %v", err) + } + if len(msg.Data) != 1 || string(msg.Data[0]) != "hello" { + t.Errorf("Data = %v, want [hello]", msg.Data) + } + if msg.Value != nil { + t.Errorf("Value = %v, want nil for a BYTES message", msg.Value) + } + if !msg.Header.Durable || msg.Header.Priority != 4 { + t.Errorf("Header = %+v, want durable priority 4", msg.Header) + } + if msg.Properties.To == nil || *msg.Properties.To != "orders" { + t.Errorf("To = %v, want orders", msg.Properties.To) + } + if got := msg.ApplicationProperties["region"]; got != "eu" { + t.Errorf("app prop region = %v, want eu", got) + } + if _, ok := msg.ApplicationProperties["__AMQ_CID"]; ok { + t.Errorf("internal __AMQ_CID leaked into application properties") + } + if got := msg.Annotations["x-opt-jms-msg-type"]; got != int8(jmsBytesMessageType) { + t.Errorf("x-opt-jms-msg-type = %v, want %d", got, jmsBytesMessageType) + } +} + +func TestCoreToAMQPTextMessage(t *testing.T) { + p := &journal.CorePayload{ + MessageID: 6, + Address: "news", + Type: journal.CoreTypeText, + Body: coreSimpleString("hi there"), + } + msg, err := coreToAMQP(p.Encode()) + if err != nil { + t.Fatalf("coreToAMQP: %v", err) + } + if msg.Value != "hi there" { + t.Errorf("Value = %v, want %q", msg.Value, "hi there") + } + if msg.Data != nil { + t.Errorf("Data = %v, want nil for a TEXT message", msg.Data) + } +} + +func TestCoreToAMQPScheduled(t *testing.T) { + p := &journal.CorePayload{ + Type: journal.CoreTypeBytes, + Body: []byte("x"), + Properties: map[string]any{scheduledMsProperty: int64(4_102_444_800_000)}, + } + msg, err := coreToAMQP(p.Encode()) + if err != nil { + t.Fatalf("coreToAMQP: %v", err) + } + if got := msg.Annotations["x-opt-delivery-time"]; got != int64(4_102_444_800_000) { + t.Errorf("x-opt-delivery-time = %v, want scheduled ms", got) + } + if _, ok := msg.ApplicationProperties[scheduledMsProperty]; ok { + t.Errorf("synthetic scheduling property leaked into application properties") + } +} + +// Message grouping (_AMQ_GROUP_ID / _AMQ_GROUP_SEQUENCE) is Artemis-internal in +// core form but maps onto standard AMQP message-properties; it must survive the +// Core->AMQP conversion rather than being stripped with the other _AMQ* keys. +func TestCoreToAMQPGroupProperties(t *testing.T) { + p := &journal.CorePayload{ + Type: journal.CoreTypeBytes, + Body: []byte("x"), + Properties: map[string]any{ + coreGroupIDProperty: "grp-core", + coreGroupSeqProperty: int32(7), + "region": "eu", + }, + } + msg, err := coreToAMQP(p.Encode()) + if err != nil { + t.Fatalf("coreToAMQP: %v", err) + } + if msg.Properties.GroupID == nil || *msg.Properties.GroupID != "grp-core" { + t.Errorf("GroupID = %v, want grp-core", msg.Properties.GroupID) + } + if msg.Properties.GroupSequence == nil || *msg.Properties.GroupSequence != 7 { + t.Errorf("GroupSequence = %v, want 7", msg.Properties.GroupSequence) + } + if _, ok := msg.ApplicationProperties[coreGroupIDProperty]; ok { + t.Errorf("_AMQ_GROUP_ID leaked into application properties") + } + if _, ok := msg.ApplicationProperties[coreGroupSeqProperty]; ok { + t.Errorf("_AMQ_GROUP_SEQUENCE leaked into application properties") + } + if got := msg.ApplicationProperties["region"]; got != "eu" { + t.Errorf("app prop region = %v, want eu", got) + } +} + +func TestCoreToAMQPUnconvertible(t *testing.T) { + if _, err := coreToAMQP([]byte{0x00, 0x01}); err == nil { + t.Fatalf("want error for a truncated core payload, got nil") + } +} diff --git a/internal/broker/drain.go b/internal/broker/drain.go new file mode 100644 index 0000000..f97fc41 --- /dev/null +++ b/internal/broker/drain.go @@ -0,0 +1,161 @@ +package broker + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/Azure/go-amqp" + "github.com/martikan/artemisctl/internal/store" +) + +// RecordSink is the persistence target for drained messages. It is +// satisfied by *store.Writer. +type RecordSink interface { + Append(store.Record) error + Sync() error +} + +// DrainQueue consumes every message currently on queue, persisting each +// batch to sink (Append + Sync/fsync) before acking the broker, so a crash +// mid-drain never loses a message: it is either still on the broker +// (unacked) or durably on disk (or both, which is safe to re-drain/replay). +// +// Idle detection: each Receive uses a per-call context with timeout idle; +// a context.DeadlineExceeded from Receive is treated as "queue is empty", +// not an error, and ends the drain — but only when the outer ctx is still +// live. If the caller's own ctx has expired/been canceled, that deadline +// propagates to the child context too, so DeadlineExceeded alone can't +// distinguish "queue idle" from "caller ran out of time"; we disambiguate +// against ctx.Err() below so a caller-timeout is reported as an error +// instead of a false "fully drained". +func (c *Client) DrainQueue(ctx context.Context, queue string, sink RecordSink, idle time.Duration, batch int) (int, error) { + if batch <= 0 { + // Credit: int32(batch) with batch<=0 grants zero link credit, which + // silently receives nothing (or reports (0,nil) on a non-empty + // queue) instead of failing loudly, so guard it here. + batch = 1 + } + recv, err := c.sess.NewReceiver(ctx, queue, &amqp.ReceiverOptions{Credit: int32(batch)}) + if err != nil { + return 0, fmt.Errorf("open receiver for %s: %w", queue, err) + } + defer recv.Close(context.Background()) + + total := 0 + pending := make([]*amqp.Message, 0, batch) + + // flush persists all currently-pending messages (Append already done + // per-message as they arrive) via Sync, then acks each one. Only after + // Sync succeeds do we ack, preserving persist-before-ack. + flush := func() error { + if len(pending) == 0 { + return nil + } + if err := sink.Sync(); err != nil { + return fmt.Errorf("sync store: %w", err) + } + for _, m := range pending { + if err := recv.AcceptMessage(ctx, m); err != nil { + return fmt.Errorf("ack message: %w", err) + } + } + pending = pending[:0] + return nil + } + + for { + rctx, cancel := context.WithTimeout(ctx, idle) + msg, err := recv.Receive(rctx, nil) + cancel() + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + if ctx.Err() != nil { + // The outer/caller context is itself done (its own + // deadline expired, or explicit cancel() was called) — + // its DeadlineExceeded/Canceled propagated into rctx, + // so this is a caller-timeout, not an idle queue. + // Returning (total, nil) here would be a false "drain + // complete" while messages remain on the broker, so + // surface it as an error instead. + return total, ctx.Err() + } + break // queue idle => drained + } + // Messages already Appended-but-not-yet-Acked when this error + // aborts the drain stay safely on the broker (no loss), but may + // be re-Appended as duplicate records on the next drain attempt + // — acceptable under the at-least-once invariant, since + // redelivery is expected to be deduped. + return total, fmt.Errorf("receive from %s: %w", queue, err) + } + + raw, err := msg.MarshalBinary() + if err != nil { + return total, fmt.Errorf("marshal message: %w", err) + } + rec := store.Record{Queue: queue, DrainedAt: time.Now().UnixNano(), AMQP: raw} + // Deterministic dedup id: derive the record UUID from a stable content + // hash rather than a random uuid. On redeliver this UUID becomes the + // broker's _AMQ_DUPL_ID, so the same broker message re-drained after a + // crash (fsync'd to the store but never acked, then re-drained) yields + // the SAME id and the broker collapses the pair to a single delivery. + // + // The hash is computed by store.DedupID over a normalized projection of + // the message that EXCLUDES the transport fields Artemis mutates on + // redelivery (Header.DeliveryCount, Header.FirstAcquirer, and the + // delivery-annotations section) — NOT over rec.AMQP. rec.AMQP keeps the + // full-fidelity bytes for a perfect replay; hashing them directly would + // break dedup because a redelivered message carries a bumped + // delivery-count that changes the marshaled bytes (empirically verified + // on Artemis 2.31.2: delivery-count 0->1 on a failed/redelivered + // delivery), yielding a different _AMQ_DUPL_ID and defeating the very + // crash-dedup this WAL exists to provide. + // + // Tradeoff (accepted): two messages that are identical apart from those + // excluded transport fields hash equal, so the broker drops one as a + // duplicate. + rec.UUID = store.DedupID(msg, queue) + // Persist BEFORE ack: append the record now; fsync+ack happens at + // batch flush, so the message is durable on disk before the broker + // is told it can be discarded. + if err := sink.Append(rec); err != nil { + return total, fmt.Errorf("append record: %w", err) + } + pending = append(pending, msg) + total++ + + if len(pending) >= batch { + if err := flush(); err != nil { + return total, err + } + } + } + if err := flush(); err != nil { + return total, err + } + return total, nil +} + +// DrainAll enumerates every queue via ListQueues and drains each one in +// turn, invoking onQueue (if non-nil) after each queue completes with the +// queue's name and the number of messages drained from it. +func (c *Client) DrainAll(ctx context.Context, sink RecordSink, idle time.Duration, batch int, onQueue func(name string, n int)) (int, error) { + queues, err := c.ListQueues(ctx) + if err != nil { + return 0, err + } + total := 0 + for _, q := range queues { + n, err := c.DrainQueue(ctx, q.Name, sink, idle, batch) + if err != nil { + return total, err + } + if onQueue != nil { + onQueue(q.Name, n) + } + total += n + } + return total, nil +} diff --git a/internal/broker/drain_dedup_integration_test.go b/internal/broker/drain_dedup_integration_test.go new file mode 100644 index 0000000..b7074ee --- /dev/null +++ b/internal/broker/drain_dedup_integration_test.go @@ -0,0 +1,216 @@ +package broker + +import ( + "bytes" + "context" + "crypto/sha256" + "path/filepath" + "testing" + "time" + + "github.com/Azure/go-amqp" + "github.com/martikan/artemisctl/internal/store" +) + +// teeSink persists each record to the underlying store AND captures it in +// memory so a test can assert on the exact records that were drained. +type teeSink struct { + w *store.Writer + recs []store.Record +} + +func (t *teeSink) Append(r store.Record) error { + t.recs = append(t.recs, r) + return t.w.Append(r) +} +func (t *teeSink) Sync() error { return t.w.Sync() } + +// TestDrainUUIDIsDeterministicContentHash is the C1 regression: the record +// UUID (replayed as _AMQ_DUPL_ID on redeliver) must be a deterministic hash of +// the AMQP bytes, not a random uuid. It asserts each drained record's UUID +// equals sha256(AMQP)[:16], and that re-draining the SAME content yields the +// SAME UUID across two independent DrainQueue runs. +func TestDrainUUIDIsDeterministicContentHash(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx) + + const queue = "dedup-det" + const body = "identical-body" + + // First drain of one message with body. + if err := sendTestMessages(ctx, c, queue, []string{body}); err != nil { + t.Fatalf("seed 1: %v", err) + } + s1 := &sliceSink{} + if _, err := c.DrainQueue(ctx, queue, s1, 3*time.Second, 10); err != nil { + t.Fatalf("drain 1: %v", err) + } + if len(s1.recs) != 1 { + t.Fatalf("want 1 record on first drain, got %d", len(s1.recs)) + } + + // UUID must equal the queue-salted content hash, not be random. + sum := sha256.New() + sum.Write(s1.recs[0].AMQP) + sum.Write([]byte{0}) + sum.Write([]byte(queue)) + want := sum.Sum(nil) + var wantUUID [16]byte + copy(wantUUID[:], want[:16]) + if s1.recs[0].UUID != wantUUID { + t.Fatalf("UUID is not sha256(AMQP||0x00||queue)[:16]: got %x want %x", s1.recs[0].UUID, wantUUID) + } + + // Second, independent drain of the SAME content -> SAME UUID. + if err := sendTestMessages(ctx, c, queue, []string{body}); err != nil { + t.Fatalf("seed 2: %v", err) + } + s2 := &sliceSink{} + if _, err := c.DrainQueue(ctx, queue, s2, 3*time.Second, 10); err != nil { + t.Fatalf("drain 2: %v", err) + } + if len(s2.recs) != 1 { + t.Fatalf("want 1 record on second drain, got %d", len(s2.recs)) + } + if s1.recs[0].UUID != s2.recs[0].UUID { + t.Fatalf("re-drained identical content produced different UUIDs: %x vs %x", + s1.recs[0].UUID, s2.recs[0].UUID) + } +} + +// TestCrossDrainSingleDelivery is the guarantee C1 restores: a message that was +// fsync'd to the store but crashed BEFORE its ack is redelivered by the broker +// and re-drained on the next run; the two drained copies must still collapse to +// exactly one delivery on redeliver. +// +// Unlike a naive "send the same body twice" test, this forces a REAL broker +// redelivery: it receives the message, records it un-acked, then settles it as +// modified/delivery-failed so Artemis redelivers it with a bumped +// Header.DeliveryCount (0 -> 1). That bump changes the marshaled AMQP bytes, so +// the two records have DIFFERENT rec.AMQP. Hashing rec.AMQP directly (the +// pre-fix behavior) would therefore give the two records DIFFERENT _AMQ_DUPL_IDs +// and the broker would NOT dedup — the exact double-delivery this WAL exists to +// prevent. store.DedupID normalizes out the volatile delivery-count, so both records +// share one id and Artemis drops the repeat. +func TestCrossDrainSingleDelivery(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx) + + const queue = "orders" + const body = "crash-and-redrain" + + path := filepath.Join(t.TempDir(), "dump.artx") + w, err := store.NewWriter(path) + if err != nil { + t.Fatal(err) + } + sink := &teeSink{w: w} + + // Seed one DURABLE message so its header section (where delivery-count + // lives) survives the round-trip and a redelivery bump is observable. + sender, err := c.sess.NewSender(ctx, queue, &amqp.SenderOptions{TargetCapabilities: []string{"queue"}}) + if err != nil { + t.Fatalf("sender: %v", err) + } + seed := amqp.NewMessage([]byte(body)) + seed.Header = &amqp.MessageHeader{Durable: true} + if err := sender.Send(ctx, seed, nil); err != nil { + t.Fatalf("seed: %v", err) + } + _ = sender.Close(context.Background()) + + // Drain #1 (crash before ack): receive the message and persist a record + // exactly as DrainQueue would (rec.AMQP = full bytes, rec.UUID = store.DedupID), + // fsync it, but do NOT ack. Then settle it modified/delivery-failed so the + // broker redelivers it with a bumped delivery-count — standing in for the + // crash-then-restart the WAL must survive. + recv, err := c.sess.NewReceiver(ctx, queue, &amqp.ReceiverOptions{Credit: 1}) + if err != nil { + t.Fatalf("receiver: %v", err) + } + rctx, rcancel := context.WithTimeout(ctx, 5*time.Second) + msg1, err := recv.Receive(rctx, nil) + rcancel() + if err != nil { + t.Fatalf("receive 1: %v", err) + } + raw1, err := msg1.MarshalBinary() + if err != nil { + t.Fatalf("marshal 1: %v", err) + } + rec1 := store.Record{Queue: queue, DrainedAt: time.Now().UnixNano(), AMQP: raw1, UUID: store.DedupID(msg1, queue)} + if err := sink.Append(rec1); err != nil { + t.Fatalf("append 1: %v", err) + } + if err := sink.Sync(); err != nil { + t.Fatalf("sync 1: %v", err) + } + if err := recv.ModifyMessage(ctx, msg1, &amqp.ModifyMessageOptions{DeliveryFailed: true}); err != nil { + t.Fatalf("modify (force redelivery): %v", err) + } + _ = recv.Close(context.Background()) + + // Drain #2 (recovery run): the real DrainQueue re-drains the REDELIVERED + // message (delivery-count now 1) into the same store and acks it. + if n, err := c.DrainQueue(ctx, queue, sink, 3*time.Second, 10); err != nil || n != 1 { + t.Fatalf("drain 2: n=%d err=%v", n, err) + } + if len(sink.recs) != 2 { + t.Fatalf("want 2 drained records, got %d", len(sink.recs)) + } + rec2 := sink.recs[1] + + // The redelivery really did mutate the wire bytes (delivery-count bump); + // this is what makes a raw sha256(rec.AMQP) fail to dedup. + if bytes.Equal(rec1.AMQP, rec2.AMQP) { + t.Fatalf("expected a real redelivery to change the AMQP bytes, but they were identical") + } + // Despite the different bytes, the dedup ids must match. + if rec1.UUID != rec2.UUID { + t.Fatalf("redelivered copy got a different dedup id: %x vs %x (raw sha256(AMQP) would give %x vs %x)", + rec1.UUID, rec2.UUID, sha256.Sum256(rec1.AMQP), sha256.Sum256(rec2.AMQP)) + } + + if err := w.Close(); err != nil { + t.Fatalf("close writer: %v", err) + } + + // Redeliver the two-record store; dedup must collapse to a single message. + if _, _, err := c.Redeliver(ctx, path, RedeliverOpts{}, nil); err != nil { + t.Fatalf("redeliver: %v", err) + } + + stats, err := c.ListQueues(ctx) + if err != nil { + t.Fatal(err) + } + var count int64 = -1 + for _, s := range stats { + if s.Name == queue { + count = s.MessageCount + } + } + if count != 1 { + t.Fatalf("cross-drain dedup failed: %s has %d messages, want 1", queue, count) + } +} diff --git a/internal/broker/drain_integration_test.go b/internal/broker/drain_integration_test.go new file mode 100644 index 0000000..b2cd989 --- /dev/null +++ b/internal/broker/drain_integration_test.go @@ -0,0 +1,144 @@ +package broker + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/martikan/artemisctl/internal/store" +) + +// sliceSink collects records in memory for assertions. +type sliceSink struct{ recs []store.Record } + +func (s *sliceSink) Append(r store.Record) error { s.recs = append(s.recs, r); return nil } +func (s *sliceSink) Sync() error { return nil } + +func TestDrainQueueRemovesMessages(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx) + + if err := sendTestMessages(ctx, c, "orders", []string{"m1", "m2", "m3"}); err != nil { + t.Fatalf("send test messages: %v", err) + } + + sink := &sliceSink{} + n, err := c.DrainQueue(ctx, "orders", sink, 3*time.Second, 10) + if err != nil { + t.Fatalf("drain: %v", err) + } + if n != 3 || len(sink.recs) != 3 { + t.Fatalf("want 3 drained, got %d/%d", n, len(sink.recs)) + } + // Second drain finds nothing. + n2, err := c.DrainQueue(ctx, "orders", &sliceSink{}, 2*time.Second, 10) + if err != nil { + t.Fatal(err) + } + if n2 != 0 { + t.Fatalf("queue not emptied, second drain got %d", n2) + } +} + +// TestDrainAllEnumeratesAndDrainsEveryQueue seeds two queues and confirms +// DrainAll walks every queue, reports each via the onQueue callback, and returns +// the combined total. +func TestDrainAllEnumeratesAndDrainsEveryQueue(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx) + + if err := sendTestMessages(ctx, c, "orders", []string{"a", "b"}); err != nil { + t.Fatalf("seed orders: %v", err) + } + if err := sendTestMessages(ctx, c, "payments", []string{"c"}); err != nil { + t.Fatalf("seed payments: %v", err) + } + + sink := &sliceSink{} + perQueue := map[string]int{} + total, err := c.DrainAll(ctx, sink, 3*time.Second, 10, func(name string, n int) { + perQueue[name] = n + }) + if err != nil { + t.Fatalf("drain all: %v", err) + } + if total != 3 || len(sink.recs) != 3 { + t.Fatalf("total drained = %d (sink %d), want 3", total, len(sink.recs)) + } + if perQueue["orders"] != 2 || perQueue["payments"] != 1 { + t.Fatalf("per-queue callback = %v, want orders:2 payments:1", perQueue) + } +} + +// TestDrainQueueRejectsExpiredCallerContext is a regression test for the +// idle-timeout vs caller-cancellation confusion: DrainQueue must not treat +// an already-expired (or canceled) OUTER ctx as "queue idle, drained". Before +// the fix, a context.DeadlineExceeded surfacing from Receive was always read +// as "queue empty" and swallowed into a (n, nil) success, even when it was +// really the caller's own deadline that had expired — silently reporting a +// drain as complete while messages could remain on the broker. +func TestDrainQueueRejectsExpiredCallerContext(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx) + + const queue = "deadline-check" + if err := sendTestMessages(ctx, c, queue, []string{"d1", "d2", "d3"}); err != nil { + t.Fatalf("send test messages: %v", err) + } + + // A parent context whose deadline has ALREADY passed, used as the OUTER + // ctx for DrainQueue. A large idle window ensures that, if this were + // mistaken for a normal idle timeout, the bug would be masked. + expiredCtx, expiredCancel := context.WithDeadline(ctx, time.Now().Add(-1*time.Second)) + defer expiredCancel() + + sink := &sliceSink{} + n, err := c.DrainQueue(expiredCtx, queue, sink, 3*time.Second, 10) + if err == nil { + t.Fatalf("want non-nil error for an already-expired caller context, got (n=%d, nil) claiming success", n) + } + if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { + t.Fatalf("want a context deadline/cancel error, got: %v", err) + } + + // Prove nothing was falsely marked as drained: the messages must still + // be sitting on the broker, retrievable via a normal, live context. + n2, err := c.DrainQueue(ctx, queue, &sliceSink{}, 3*time.Second, 10) + if err != nil { + t.Fatalf("follow-up drain: %v", err) + } + if n2 != 3 { + t.Fatalf("want all 3 messages still present after the false-timeout was rejected, got %d", n2) + } +} diff --git a/internal/broker/health.go b/internal/broker/health.go new file mode 100644 index 0000000..d4c429c --- /dev/null +++ b/internal/broker/health.go @@ -0,0 +1,123 @@ +package broker + +import ( + "context" + "encoding/json" + "fmt" +) + +// Verdict is the single-word health rating derived from a broker's disk, +// memory, and producer-blocking state. See classify for the thresholds. +type Verdict string + +const ( + OK Verdict = "OK" // all usage below 70%, not blocking + Degraded Verdict = "DEGRADED" // some usage in 70–90%, still serving + Critical Verdict = "CRITICAL" // usage above 90% or producers are blocked +) + +// Health is a point-in-time snapshot of a broker's resource pressure. Disk and +// memory are percentages (0–100); Blocking is true when the broker is refusing +// producers because of disk-full protection; Verdict is the classify result. +type Health struct { + DiskUsagePct float64 + MemoryUsagePct float64 + Blocking bool + Verdict Verdict +} + +// classify maps raw usage percentages and the blocking flag to a Verdict: +// Critical if blocking or either usage exceeds 90%, Degraded if either reaches +// 70%, otherwise OK. +func classify(diskPct, memPct float64, blocking bool) Verdict { + if blocking || diskPct > 90 || memPct > 90 { + return Critical + } + if diskPct >= 70 || memPct >= 70 { + return Degraded + } + return OK +} + +// scalarReply parses Artemis's array-wrapped scalar management reply into a float64. +// +// Confirmed against apache/activemq-artemis:2.31.2 in Task 8's spike +// (internal/broker/probe_test.go, run and deleted after verification): +// - broker.getDiskStoreUsage returns reply.Value as the string "[0.0]" — a +// JSON-array-wrapped float64 fraction in 0..1 (fresh broker: 0.0). +// - broker.getAddressMemoryUsagePercentage returns reply.Value as the +// string "[0]" — a JSON-array-wrapped number already in 0..100 percent +// (fresh broker: 0). +// +// Both op names and shapes matched the brief's candidates exactly; no +// adjustment to the operation names or parsing was required. +func scalarReply(reply interface{}) (float64, error) { + s, ok := reply.(string) + if !ok { + return 0, fmt.Errorf("unexpected scalar reply type %T", reply) + } + var arr []float64 + if err := json.Unmarshal([]byte(s), &arr); err != nil { + return 0, fmt.Errorf("parse scalar reply %q: %w", s, err) + } + if len(arr) == 0 { + return 0, fmt.Errorf("empty scalar reply") + } + return arr[0], nil +} + +// CheckHealth queries the broker's disk-store usage, address-memory usage, and +// producer-blocking state over the management address and returns a Health +// snapshot with the classified Verdict. Blocking is inferred from the broker's +// own max-disk-usage threshold (see the disk-full logic below), so it fires +// under a custom lower limit too. +func (c *Client) CheckHealth(ctx context.Context) (Health, error) { + diskReply, err := c.callManagement(ctx, "broker", "getDiskStoreUsage", "[]") + if err != nil { + return Health{}, err + } + diskFrac, err := scalarReply(diskReply.Value) + if err != nil { + return Health{}, err + } + memReply, err := c.callManagement(ctx, "broker", "getAddressMemoryUsagePercentage", "[]") + if err != nil { + return Health{}, err + } + memPct, err := scalarReply(memReply.Value) + if err != nil { + return Health{}, err + } + diskPct := diskFrac * 100 // getDiskStoreUsage returns a 0..1 fraction + + // I1 — real producer-blocking (disk-full block). + // + // Artemis's disk-full protection blocks ALL producers once disk store + // usage crosses the configured max-disk-usage. That is genuine producer + // blocking, not an arbitrary threshold. The dedicated broker.isDiskFull + // operation does NOT exist on apache/activemq-artemis:2.31.2 (probed in + // the I1 spike: reply "AMQ229069: no operation isDiskFull/0"), so we read + // the configured threshold and compare it against current usage. + // + // Confirmed op + reply shape (I1 spike, run and deleted): + // broker.getMaxDiskUsage -> reply.Value string "[90]" — a + // JSON-array-wrapped percent (0..100); the block threshold in percent. + // (broker.getDiskStoreUsage already gives current usage as a 0..1 + // fraction, above.) Blocking is true when disk usage has reached the + // broker's own block threshold, so it tracks the real config rather than + // the hardcoded classify() cutoffs — e.g. it fires at a custom + // max-disk-usage of 70 that the >90 rule would miss. + maxDiskReply, err := c.callManagement(ctx, "broker", "getMaxDiskUsage", "[]") + if err != nil { + return Health{}, err + } + maxDiskPct, err := scalarReply(maxDiskReply.Value) + if err != nil { + return Health{}, err + } + blocking := maxDiskPct > 0 && diskPct >= maxDiskPct + + h := Health{DiskUsagePct: diskPct, MemoryUsagePct: memPct, Blocking: blocking} + h.Verdict = classify(h.DiskUsagePct, h.MemoryUsagePct, h.Blocking) + return h, nil +} diff --git a/internal/broker/health_integration_test.go b/internal/broker/health_integration_test.go new file mode 100644 index 0000000..6939348 --- /dev/null +++ b/internal/broker/health_integration_test.go @@ -0,0 +1,33 @@ +package broker + +import ( + "context" + "testing" + "time" +) + +func TestCheckHealthIntegration(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx) + h, err := c.CheckHealth(ctx) + if err != nil { + t.Fatalf("health: %v", err) + } + if h.Verdict != OK { + t.Fatalf("fresh broker should be OK, got %s (disk %.1f mem %.1f)", h.Verdict, h.DiskUsagePct, h.MemoryUsagePct) + } + // I1: a fresh broker with a near-empty disk is well below max-disk-usage, + // so the disk-full producer-block indicator must be false. + if h.Blocking { + t.Fatalf("fresh broker should not report producer blocking (disk %.1f%%)", h.DiskUsagePct) + } +} diff --git a/internal/broker/health_test.go b/internal/broker/health_test.go new file mode 100644 index 0000000..35b8340 --- /dev/null +++ b/internal/broker/health_test.go @@ -0,0 +1,47 @@ +package broker + +import "testing" + +func TestScalarReply(t *testing.T) { + t.Run("array-wrapped float", func(t *testing.T) { + got, err := scalarReply("[0.42]") + if err != nil || got != 0.42 { + t.Fatalf("got %v, %v; want 0.42, nil", got, err) + } + }) + t.Run("non-string reply", func(t *testing.T) { + if _, err := scalarReply(123); err == nil { + t.Fatal("want error for non-string reply") + } + }) + t.Run("unparseable string", func(t *testing.T) { + if _, err := scalarReply("not-json"); err == nil { + t.Fatal("want error for unparseable reply") + } + }) + t.Run("empty array", func(t *testing.T) { + if _, err := scalarReply("[]"); err == nil { + t.Fatal("want error for empty array reply") + } + }) +} + +func TestClassify(t *testing.T) { + cases := []struct { + disk, mem float64 + blocking bool + want Verdict + }{ + {10, 10, false, OK}, + {75, 10, false, Degraded}, + {10, 85, false, Degraded}, + {95, 10, false, Critical}, + {10, 10, true, Critical}, + {80, 92, false, Critical}, + } + for _, c := range cases { + if got := classify(c.disk, c.mem, c.blocking); got != c.want { + t.Fatalf("classify(%v,%v,%v)=%s want %s", c.disk, c.mem, c.blocking, got, c.want) + } + } +} diff --git a/internal/broker/management.go b/internal/broker/management.go new file mode 100644 index 0000000..e36157d --- /dev/null +++ b/internal/broker/management.go @@ -0,0 +1,100 @@ +package broker + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/Azure/go-amqp" +) + +// QueueStat is one queue's name and current message depth, as reported by the +// broker's listQueues management operation. MessageCount arrives as a +// JSON string and is decoded into an int64. +type QueueStat struct { + Name string `json:"name"` + MessageCount int64 `json:"messageCount,string"` +} + +// callManagement performs a request/reply against activemq.management. +func (c *Client) callManagement(ctx context.Context, resource, operation, body string) (*amqp.Message, error) { + recv, err := c.sess.NewReceiver(ctx, "", &amqp.ReceiverOptions{DynamicAddress: true}) + if err != nil { + return nil, fmt.Errorf("create reply receiver: %w", err) + } + defer recv.Close(context.Background()) + replyTo := recv.Address() + + sender, err := c.sess.NewSender(ctx, "activemq.management", nil) + if err != nil { + return nil, fmt.Errorf("create management sender: %w", err) + } + defer sender.Close(context.Background()) + + msg := &amqp.Message{ + Value: body, + Properties: &amqp.MessageProperties{ReplyTo: &replyTo}, + ApplicationProperties: map[string]interface{}{ + "_AMQ_ResourceName": resource, + "_AMQ_OperationName": operation, + }, + } + if err := sender.Send(ctx, msg, nil); err != nil { + return nil, fmt.Errorf("send management request: %w", err) + } + reply, err := recv.Receive(ctx, nil) + if err != nil { + return nil, fmt.Errorf("receive management reply: %w", err) + } + _ = recv.AcceptMessage(ctx, reply) + if ok, present := reply.ApplicationProperties["_AMQ_OperationSucceeded"].(bool); present && !ok { + return nil, fmt.Errorf("broker rejected %s.%s: %v", resource, operation, reply.Value) + } + return reply, nil +} + +// ListQueues returns the broker's user queues with their message counts, +// sorted by depth descending, with internal and temporary queues filtered out. +// It asks the broker for up to 1000 queues in a single management call. +func (c *Client) ListQueues(ctx context.Context) ([]QueueStat, error) { + filter := `{"field":"","operation":"","value":"","sortField":"messageCount","sortOrder":"desc"}` + body := fmt.Sprintf(`[%q, 1, 1000]`, filter) + reply, err := c.callManagement(ctx, "broker", "listQueues", body) + if err != nil { + return nil, err + } + return parseQueueStatsReply(reply) +} + +func parseQueueStatsReply(reply *amqp.Message) ([]QueueStat, error) { + strVal, ok := reply.Value.(string) + if !ok { + return nil, fmt.Errorf("unexpected management response format") + } + var outer []string + if err := json.Unmarshal([]byte(strVal), &outer); err != nil { + return nil, fmt.Errorf("parse outer array: %w", err) + } + if len(outer) == 0 { + return nil, nil + } + var paged struct { + Data []QueueStat `json:"data"` + } + if err := json.Unmarshal([]byte(outer[0]), &paged); err != nil { + return nil, fmt.Errorf("parse paged response: %w", err) + } + return filterInternalQueues(paged.Data), nil +} + +func filterInternalQueues(in []QueueStat) []QueueStat { + var out []QueueStat + for _, q := range in { + if strings.HasPrefix(q.Name, "activemq.") || strings.HasPrefix(q.Name, "$") || len(q.Name) == 36 { + continue + } + out = append(out, q) + } + return out +} diff --git a/internal/broker/management_integration_test.go b/internal/broker/management_integration_test.go new file mode 100644 index 0000000..0f493ae --- /dev/null +++ b/internal/broker/management_integration_test.go @@ -0,0 +1,124 @@ +package broker + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/Azure/go-amqp" + + "github.com/martikan/artemisctl/internal/brokertest" + "github.com/martikan/artemisctl/internal/store" +) + +// sendTestMessages sends the given bodies to the named queue over the +// client's existing AMQP session, forcing Artemis to auto-create the +// queue and populate its message count. +func sendTestMessages(ctx context.Context, c *Client, queue string, bodies []string) error { + // TargetCapabilities: []string{"queue"} tells Artemis to route this as an + // anycast queue rather than the default multicast address, so the queue + // is auto-created and picks up a message count even with no consumer + // attached. + sender, err := c.Session().NewSender(ctx, queue, &amqp.SenderOptions{TargetCapabilities: []string{"queue"}}) + if err != nil { + return err + } + defer sender.Close(context.Background()) + for i, body := range bodies { + msg := amqp.NewMessage([]byte(body)) + // Real producers (notably JMS) always stamp an AMQP message-id; browse + // reports it as BrowsedMessage.ID and BrowseMessage matches on it, so + // give each test message a unique one instead of leaving it nil. + msg.Properties = &amqp.MessageProperties{MessageID: fmt.Sprintf("%s-%d-%s", queue, i, body)} + if err := sender.Send(ctx, msg, nil); err != nil { + return err + } + } + return nil +} + +// startArtemis returns connection props for the shared integration broker, +// resetting it to a clean slate first. Takes testing.TB so both tests and +// benchmarks can use it. +func startArtemis(t testing.TB) ConnectionProps { + t.Helper() + sc := brokertest.Shared(t) + props := ConnectionProps{URL: sc.URL, Username: sc.Username, Password: sc.Password} + resetBroker(t, props) + return props +} + +// discardSink drops every drained record; used only to purge queues. +type discardSink struct{} + +func (discardSink) Append(store.Record) error { return nil } +func (discardSink) Sync() error { return nil } + +// resetBroker returns the shared, reused broker to a clean slate so each test +// starts fresh despite dirty-context reuse: it lifts any wildcard cordon a +// crashed cordon test may have left (which would otherwise reject the next +// test's producers), then drains every user queue empty. +func resetBroker(t testing.TB, props ConnectionProps) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatalf("reset connect: %v", err) + } + defer c.Close(ctx) + // Lift any leftover cordon by overwriting the wildcard with permissive + // settings. removeAddressSettings("#") does not reliably clear it on 2.42. + _ = c.Uncordon(ctx, brokertest.PermissiveWildcardSettings) + qs, err := c.ListQueues(ctx) + if err != nil { + t.Fatalf("reset list queues: %v", err) + } + for _, q := range qs { + if _, err := c.DrainQueue(ctx, q.Name, discardSink{}, 500*time.Millisecond, 200); err != nil { + t.Fatalf("reset drain %s: %v", q.Name, err) + } + } +} + +func TestListQueuesIntegration(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer c.Close(ctx) + + if err := sendTestMessages(ctx, c, "orders", []string{"order-1", "order-2"}); err != nil { + t.Fatalf("send test messages: %v", err) + } + + stats, err := c.ListQueues(ctx) + if err != nil { + t.Fatalf("list queues: %v", err) + } + + var found *QueueStat + for i := range stats { + s := stats[i] + if strings.HasPrefix(s.Name, "activemq.") || strings.HasPrefix(s.Name, "$") { + t.Fatalf("filter did not hold against live broker, got internal queue: %+v", s) + } + if s.Name == "orders" { + found = &s + } + } + if found == nil { + t.Fatalf("expected queue %q in results, got %+v", "orders", stats) + } + if found.MessageCount < 2 { + t.Fatalf("expected MessageCount >= 2 for %q, got %d", "orders", found.MessageCount) + } +} diff --git a/internal/broker/management_test.go b/internal/broker/management_test.go new file mode 100644 index 0000000..a4be2ca --- /dev/null +++ b/internal/broker/management_test.go @@ -0,0 +1,89 @@ +package broker + +import ( + "encoding/json" + "testing" + + "github.com/Azure/go-amqp" +) + +func TestParseQueueStatsReply(t *testing.T) { + t.Run("double-encoded JSON with quoted messageCount", func(t *testing.T) { + inner := `{"data":[{"name":"orders","messageCount":"7"},{"name":"activemq.notifications","messageCount":"1"}]}` + outerBytes, err := json.Marshal([]string{inner}) + if err != nil { + t.Fatalf("marshal outer: %v", err) + } + msg := &amqp.Message{Value: string(outerBytes)} + + got, err := parseQueueStatsReply(msg) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 1 { + t.Fatalf("expected exactly one queue stat, got %+v", got) + } + want := QueueStat{Name: "orders", MessageCount: 7} + if got[0] != want { + t.Fatalf("got %+v want %+v", got[0], want) + } + }) + + t.Run("empty data set returns nil without error", func(t *testing.T) { + inner := `{"data":[]}` + outerBytes, err := json.Marshal([]string{inner}) + if err != nil { + t.Fatalf("marshal outer: %v", err) + } + msg := &amqp.Message{Value: string(outerBytes)} + + got, err := parseQueueStatsReply(msg) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 0 { + t.Fatalf("expected no queue stats, got %+v", got) + } + }) + + t.Run("empty outer array returns nil without error", func(t *testing.T) { + msg := &amqp.Message{Value: `[]`} + got, err := parseQueueStatsReply(msg) + if err != nil || got != nil { + t.Fatalf("got %+v, %v; want nil, nil", got, err) + } + }) + + t.Run("non-string reply value errors", func(t *testing.T) { + if _, err := parseQueueStatsReply(&amqp.Message{Value: 123}); err == nil { + t.Fatal("want error for non-string management reply") + } + }) + + t.Run("malformed outer JSON errors", func(t *testing.T) { + if _, err := parseQueueStatsReply(&amqp.Message{Value: `not-json`}); err == nil { + t.Fatal("want error for malformed outer array") + } + }) + + t.Run("malformed paged JSON errors", func(t *testing.T) { + outer, _ := json.Marshal([]string{`{"data": not-json}`}) + if _, err := parseQueueStatsReply(&amqp.Message{Value: string(outer)}); err == nil { + t.Fatal("want error for malformed paged response") + } + }) +} + +func TestFilterInternalQueues(t *testing.T) { + in := []QueueStat{ + {Name: "orders", MessageCount: 5}, + {Name: "activemq.notifications", MessageCount: 1}, + {Name: "$sys.foo", MessageCount: 1}, + {Name: "123e4567-e89b-12d3-a456-426614174000", MessageCount: 1}, // 36 chars + {Name: "payments", MessageCount: 2}, + } + got := filterInternalQueues(in) + if len(got) != 2 || got[0].Name != "orders" || got[1].Name != "payments" { + t.Fatalf("unexpected filter result: %+v", got) + } +} diff --git a/internal/broker/produce.go b/internal/broker/produce.go new file mode 100644 index 0000000..29fdae4 --- /dev/null +++ b/internal/broker/produce.go @@ -0,0 +1,327 @@ +// The produce command injects messages onto a queue for testing and load +// generation. Two sources feed it, both of which resolve to a slice of +// *amqp.Message before anything is sent: +// +// - GenerateMessages builds synthetic messages of a fixed body size. +// - ParseArtemisMessages reads a JSON array in the Artemis web-console +// layout (the same shape listMessagesAsJSON emits, with typed property +// buckets like StringProperties/IntProperties), so a message copied out +// of the console can be replayed verbatim. +// +// Produce then sends the slice to a single queue with an optional +// messages-per-second throttle, honoring context cancellation (SIGINT) so a +// long run stops cleanly between messages. + +package broker + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "sync" + "sync/atomic" + "time" + + "github.com/Azure/go-amqp" + "github.com/google/uuid" +) + +// defaultPriority is the AMQP/Artemis default message priority (0-9) used when +// a message does not specify one. +const defaultPriority uint8 = 4 + +// bodyAlphabet fills the random tail of a generated message body. +const bodyAlphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +// GenerateMessages builds count synthetic durable messages, each with a body +// of exactly size bytes: a "msg--" prefix followed by random alphanumeric +// padding (truncated to size if the prefix alone already exceeds it). Each +// message is stamped with a unique AMQP message-id ("gen--") so browse +// reports it in the ID column and BrowseMessage can drill into it -- Artemis +// does not synthesize an AMQP message-id for a send that omits one. The +// supplied application properties are copied onto every message. +func GenerateMessages(count, size int, props map[string]interface{}) []*amqp.Message { + if count < 0 { + count = 0 + } + if size < 0 { + size = 0 + } + msgs := make([]*amqp.Message, 0, count) + for i := 0; i < count; i++ { + body := make([]byte, size) + n := copy(body, fmt.Sprintf("msg-%d-", i)) + for j := n; j < size; j++ { + body[j] = bodyAlphabet[rand.Intn(len(bodyAlphabet))] + } + msg := amqp.NewMessage(body) + msg.Header = &amqp.MessageHeader{Durable: true, Priority: defaultPriority} + msg.Properties = &amqp.MessageProperties{MessageID: fmt.Sprintf("gen-%d-%s", i, uuid.NewString())} + if len(props) > 0 { + msg.ApplicationProperties = copyProps(props) + } + msgs = append(msgs, msg) + } + return msgs +} + +// artemisMessage mirrors the fields of the Artemis web-console / management +// listMessagesAsJSON message layout that are meaningful when re-producing a +// message. messageID/timestamp/userID are intentionally omitted: the broker +// assigns those on send, so echoing them back would be meaningless. The typed +// property buckets each map a property name to a value of the corresponding +// AMQP type. +type artemisMessage struct { + Durable *bool `json:"durable"` + Priority *uint8 `json:"priority"` + Expiration int64 `json:"expiration"` + Text string `json:"text"` + + StringProperties map[string]string `json:"StringProperties"` + BooleanProperties map[string]bool `json:"BooleanProperties"` + ByteProperties map[string]int8 `json:"ByteProperties"` + ShortProperties map[string]int16 `json:"ShortProperties"` + IntProperties map[string]int32 `json:"IntProperties"` + LongProperties map[string]int64 `json:"LongProperties"` + FloatProperties map[string]float32 `json:"FloatProperties"` + DoubleProperties map[string]float64 `json:"DoubleProperties"` +} + +// ParseArtemisMessages decodes a JSON array of Artemis-console-shaped messages +// into amqp.Messages ready to send. The message body is taken from the "text" +// field; "durable" defaults to true when absent, "priority" to 4; a non-zero +// "expiration" (unix millis) becomes the AMQP absolute expiry time. Every typed +// property bucket is flattened into the message's application properties, and +// the supplied extra properties are merged on top (overriding any file property +// of the same name). The "address"/"type" fields are ignored — the caller's +// target queue is authoritative and the body is always sent as a data section. +func ParseArtemisMessages(data []byte, extra map[string]interface{}) ([]*amqp.Message, error) { + var raw []artemisMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("parse message file: %w", err) + } + msgs := make([]*amqp.Message, 0, len(raw)) + for _, am := range raw { + msg := amqp.NewMessage([]byte(am.Text)) + + durable := true + if am.Durable != nil { + durable = *am.Durable + } + priority := defaultPriority + if am.Priority != nil { + priority = *am.Priority + } + msg.Header = &amqp.MessageHeader{Durable: durable, Priority: priority} + + if am.Expiration > 0 { + expiry := time.UnixMilli(am.Expiration) + msg.Properties = &amqp.MessageProperties{AbsoluteExpiryTime: &expiry} + } + + props := map[string]interface{}{} + for k, v := range am.StringProperties { + props[k] = v + } + for k, v := range am.BooleanProperties { + props[k] = v + } + for k, v := range am.ByteProperties { + props[k] = v + } + for k, v := range am.ShortProperties { + props[k] = v + } + for k, v := range am.IntProperties { + props[k] = v + } + for k, v := range am.LongProperties { + props[k] = v + } + for k, v := range am.FloatProperties { + props[k] = v + } + for k, v := range am.DoubleProperties { + props[k] = v + } + for k, v := range extra { + props[k] = v + } + if len(props) > 0 { + msg.ApplicationProperties = props + } + msgs = append(msgs, msg) + } + return msgs, nil +} + +// copyProps returns a shallow copy of src so callers that mutate one message's +// properties do not affect others sharing the same source map. +func copyProps(src map[string]interface{}) map[string]interface{} { + dst := make(map[string]interface{}, len(src)) + for k, v := range src { + dst[k] = v + } + return dst +} + +// Produce sends msgs to queue, optionally throttled to ratePerSec messages +// per second in total (ratePerSec <= 0 sends as fast as possible). workers is +// the number of parallel senders: 1 (or less) sends sequentially over a +// single sender, preserving order; more than 1 opens one sender per worker, +// each on its own AMQP session, so settlement round-trips overlap — this +// raises throughput but does NOT preserve delivery order. It stops early and +// returns the count sent so far, together with ctx.Err(), if the context is +// canceled (e.g. SIGINT). onProgress, when non-nil, is called after each +// successful send with the running total; with workers > 1 it may be called +// from multiple goroutines concurrently. +func (c *Client) Produce(ctx context.Context, queue string, msgs []*amqp.Message, ratePerSec, workers int, onProgress func(sent int)) (int, error) { + if workers <= 1 { + return c.produceSequential(ctx, queue, msgs, ratePerSec, onProgress) + } + return c.produceParallel(ctx, queue, msgs, ratePerSec, workers, onProgress) +} + +// produceSequential is the ordered single-sender path. +func (c *Client) produceSequential(ctx context.Context, queue string, msgs []*amqp.Message, ratePerSec int, onProgress func(sent int)) (int, error) { + // TargetCapabilities: []string{"queue"} routes as an anycast queue rather + // than the default multicast address; without it messages never land in + // the queue (see redeliver.go / sendTestMessages for the same requirement). + sender, err := c.sess.NewSender(ctx, queue, &amqp.SenderOptions{TargetCapabilities: []string{"queue"}}) + if err != nil { + return 0, fmt.Errorf("open sender for %s: %w", queue, err) + } + defer sender.Close(context.Background()) + + var interval time.Duration + if ratePerSec > 0 { + interval = time.Second / time.Duration(ratePerSec) + } + + sent := 0 + for i, msg := range msgs { + if err := ctx.Err(); err != nil { + return sent, err + } + if interval > 0 && i > 0 { + select { + case <-ctx.Done(): + return sent, ctx.Err() + case <-time.After(interval): + } + } + if err := sender.Send(ctx, msg, nil); err != nil { + return sent, fmt.Errorf("send to %s: %w", queue, err) + } + sent++ + if onProgress != nil { + onProgress(sent) + } + } + return sent, nil +} + +// produceParallel fans msgs out to workers senders, each on its own session +// so the per-link transfer serialization in go-amqp doesn't gate throughput — +// each worker's settlement wait overlaps the others'. A single shared ticker +// keeps ratePerSec global across workers. The first send error (or ctx +// cancellation) cancels the remaining workers; the returned count is the +// number of messages actually settled. +func (c *Client) produceParallel(ctx context.Context, queue string, msgs []*amqp.Message, ratePerSec, workers int, onProgress func(sent int)) (int, error) { + if workers > len(msgs) { + workers = len(msgs) + } + + senders := make([]*amqp.Sender, 0, workers) + defer func() { + for _, s := range senders { + _ = s.Close(context.Background()) + } + }() + for i := 0; i < workers; i++ { + sess, err := c.conn.NewSession(ctx, nil) + if err != nil { + return 0, fmt.Errorf("open session %d: %w", i, err) + } + // Anycast routing, same requirement as the sequential path. + s, err := sess.NewSender(ctx, queue, &amqp.SenderOptions{TargetCapabilities: []string{"queue"}}) + if err != nil { + return 0, fmt.Errorf("open sender %d for %s: %w", i, queue, err) + } + senders = append(senders, s) + } + + // Global throttle: one ticker shared by all workers, so ratePerSec is the + // total rate, not per-worker. Receiving from the shared channel naturally + // distributes send slots across workers. + var tick <-chan time.Time + if ratePerSec > 0 { + ticker := time.NewTicker(time.Second / time.Duration(ratePerSec)) + defer ticker.Stop() + tick = ticker.C + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + work := make(chan *amqp.Message) + go func() { + defer close(work) + for _, m := range msgs { + select { + case work <- m: + case <-ctx.Done(): + return + } + } + }() + + var ( + sent atomic.Int64 + wg sync.WaitGroup + errOnce sync.Once + firstErr error + ) + fail := func(err error) { + errOnce.Do(func() { firstErr = err }) + cancel() + } + + for _, sender := range senders { + wg.Add(1) + go func(sender *amqp.Sender) { + defer wg.Done() + for msg := range work { + if tick != nil { + select { + case <-tick: + case <-ctx.Done(): + fail(ctx.Err()) + return + } + } + if err := sender.Send(ctx, msg, nil); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + fail(ctxErr) + } else { + fail(fmt.Errorf("send to %s: %w", queue, err)) + } + return + } + n := sent.Add(1) + if onProgress != nil { + onProgress(int(n)) + } + } + }(sender) + } + wg.Wait() + + if firstErr == nil { + // Workers exited via a closed work channel, but the feeder also closes + // it on cancellation — surface the cancellation if that's what happened. + firstErr = ctx.Err() + } + return int(sent.Load()), firstErr +} diff --git a/internal/broker/produce_bench_test.go b/internal/broker/produce_bench_test.go new file mode 100644 index 0000000..f0140c1 --- /dev/null +++ b/internal/broker/produce_bench_test.go @@ -0,0 +1,124 @@ +// Benchmarks measuring produce throughput against a real Artemis broker +// (Testcontainers). Run via `make bench`; skipped in -short and never run by +// plain `go test` (benchmarks need -bench). +// +// BenchmarkProduce answers "do parallel workers help?": each worker is a +// sender on its own session over ONE shared connection, so settlement +// round-trips overlap. BenchmarkProduceMultiConn answers "do separate +// connections beat sessions?": same worker counts, but each worker gets a +// whole connection of its own. +package broker + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/go-amqp" +) + +var benchWorkerCounts = []int{1, 2, 4, 8, 16} + +// benchQueue returns a unique queue name per sub-benchmark run so message +// depth from earlier runs doesn't skew later ones. +var benchQueueSeq atomic.Int64 + +func benchQueue(prefix string) string { + return fmt.Sprintf("%s-%d", prefix, benchQueueSeq.Add(1)) +} + +func BenchmarkProduce(b *testing.B) { + if testing.Short() { + b.Skip("needs broker") + } + props := startArtemis(b) + + for _, w := range benchWorkerCounts { + b.Run(fmt.Sprintf("workers=%d", w), func(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + b.Fatalf("connect: %v", err) + } + defer c.Close(ctx) + + msgs := GenerateMessages(b.N, 256, nil) + queue := benchQueue("bench-produce") + + b.ResetTimer() + sent, err := c.Produce(ctx, queue, msgs, 0, w, nil) + b.StopTimer() + if err != nil { + b.Fatalf("produce: %v", err) + } + if sent != b.N { + b.Fatalf("sent = %d, want %d", sent, b.N) + } + b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "msgs/s") + }) + } +} + +func BenchmarkProduceMultiConn(b *testing.B) { + if testing.Short() { + b.Skip("needs broker") + } + props := startArtemis(b) + + for _, w := range benchWorkerCounts { + b.Run(fmt.Sprintf("conns=%d", w), func(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + clients := make([]*Client, w) + for i := range clients { + c, err := Connect(ctx, props) + if err != nil { + b.Fatalf("connect %d: %v", i, err) + } + defer c.Close(ctx) + clients[i] = c + } + + msgs := GenerateMessages(b.N, 256, nil) + queue := benchQueue("bench-produce-multiconn") + + // Split messages across clients; each client sends its slice + // sequentially over its own connection (workers=1), so the only + // parallelism measured is connection-level. + b.ResetTimer() + var wg sync.WaitGroup + var sent atomic.Int64 + errs := make([]error, w) + for i, c := range clients { + lo := i * len(msgs) / w + hi := (i + 1) * len(msgs) / w + if lo == hi { + continue + } + wg.Add(1) + go func(i int, c *Client, part []*amqp.Message) { + defer wg.Done() + n, err := c.Produce(ctx, queue, part, 0, 1, nil) + sent.Add(int64(n)) + errs[i] = err + }(i, c, msgs[lo:hi]) + } + wg.Wait() + b.StopTimer() + for i, err := range errs { + if err != nil { + b.Fatalf("produce conn %d: %v", i, err) + } + } + if got := int(sent.Load()); got != b.N { + b.Fatalf("sent = %d, want %d", got, b.N) + } + b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "msgs/s") + }) + } +} diff --git a/internal/broker/produce_test.go b/internal/broker/produce_test.go new file mode 100644 index 0000000..8d6a1f7 --- /dev/null +++ b/internal/broker/produce_test.go @@ -0,0 +1,313 @@ +package broker + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" +) + +func TestGenerateMessagesBodySizeAndProps(t *testing.T) { + props := map[string]interface{}{"region": "eu"} + msgs := GenerateMessages(3, 64, props) + if len(msgs) != 3 { + t.Fatalf("want 3 messages, got %d", len(msgs)) + } + for i, m := range msgs { + if got := len(m.GetData()); got != 64 { + t.Fatalf("msg %d body size = %d, want 64", i, got) + } + if m.Header == nil || !m.Header.Durable { + t.Fatalf("msg %d not durable", i) + } + if m.ApplicationProperties["region"] != "eu" { + t.Fatalf("msg %d missing property, got %v", i, m.ApplicationProperties) + } + } + // Distinct property maps: mutating one must not touch another. + msgs[0].ApplicationProperties["region"] = "us" + if msgs[1].ApplicationProperties["region"] != "eu" { + t.Fatalf("property maps are shared across generated messages") + } +} + +func TestGenerateMessagesTinySize(t *testing.T) { + // size smaller than the "msg--" prefix must still yield exactly size bytes. + msgs := GenerateMessages(1, 2, nil) + if got := len(msgs[0].GetData()); got != 2 { + t.Fatalf("body size = %d, want 2", got) + } +} + +func TestGenerateMessagesClampsNegative(t *testing.T) { + // Negative count and size must clamp to 0 rather than panic. + if msgs := GenerateMessages(-5, -1, nil); len(msgs) != 0 { + t.Fatalf("negative count: got %d messages, want 0", len(msgs)) + } + msgs := GenerateMessages(1, -1, nil) + if len(msgs) != 1 || len(msgs[0].GetData()) != 0 { + t.Fatalf("negative size: got %d msgs, body %d, want 1 msg / 0 bytes", len(msgs), len(msgs[0].GetData())) + } +} + +func TestParseArtemisMessagesInvalidJSON(t *testing.T) { + if _, err := ParseArtemisMessages([]byte("{not an array"), nil); err == nil { + t.Fatal("want parse error for malformed JSON") + } +} + +// TestParseArtemisMessagesRemainingBuckets covers the Byte/Short/Float/Double +// typed-property buckets the primary test doesn't touch. +func TestParseArtemisMessagesRemainingBuckets(t *testing.T) { + data := []byte(`[{ + "text": "x", + "ByteProperties": { "b": 7 }, + "ShortProperties": { "s": 300 }, + "FloatProperties": { "f": 1.5 }, + "DoubleProperties": { "d": 2.5 } + }]`) + msgs, err := ParseArtemisMessages(data, nil) + if err != nil { + t.Fatalf("parse: %v", err) + } + ap := msgs[0].ApplicationProperties + if v, ok := ap["b"].(int8); !ok || v != 7 { + t.Fatalf("ByteProperties b = %#v, want int8(7)", ap["b"]) + } + if v, ok := ap["s"].(int16); !ok || v != 300 { + t.Fatalf("ShortProperties s = %#v, want int16(300)", ap["s"]) + } + if v, ok := ap["f"].(float32); !ok || v != 1.5 { + t.Fatalf("FloatProperties f = %#v, want float32(1.5)", ap["f"]) + } + if v, ok := ap["d"].(float64); !ok || v != 2.5 { + t.Fatalf("DoubleProperties d = %#v, want float64(2.5)", ap["d"]) + } +} + +func TestParseArtemisMessagesTypedProps(t *testing.T) { + data := []byte(`[ + { + "address": "orders", + "durable": true, + "priority": 7, + "expiration": 1700000000000, + "type": 3, + "text": "hello", + "StringProperties": { "region": "eu" }, + "IntProperties": { "n": 42 }, + "BooleanProperties": { "flag": true }, + "LongProperties": { "big": 9000000000 } + }, + { "text": "second" } + ]`) + msgs, err := ParseArtemisMessages(data, map[string]interface{}{"injected": "x"}) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(msgs) != 2 { + t.Fatalf("want 2 messages, got %d", len(msgs)) + } + + m := msgs[0] + if string(m.GetData()) != "hello" { + t.Fatalf("body = %q, want hello", string(m.GetData())) + } + if m.Header == nil || !m.Header.Durable || m.Header.Priority != 7 { + t.Fatalf("header = %+v, want durable priority 7", m.Header) + } + if m.Properties == nil || m.Properties.AbsoluteExpiryTime == nil { + t.Fatalf("expiration not mapped to absolute expiry time") + } + if !m.Properties.AbsoluteExpiryTime.Equal(time.UnixMilli(1700000000000)) { + t.Fatalf("expiry = %v, want %v", m.Properties.AbsoluteExpiryTime, time.UnixMilli(1700000000000)) + } + // Typed property buckets must land as their Go types, not float64. + if v, ok := m.ApplicationProperties["n"].(int32); !ok || v != 42 { + t.Fatalf("IntProperties n = %#v, want int32(42)", m.ApplicationProperties["n"]) + } + if v, ok := m.ApplicationProperties["big"].(int64); !ok || v != 9000000000 { + t.Fatalf("LongProperties big = %#v, want int64(9000000000)", m.ApplicationProperties["big"]) + } + if m.ApplicationProperties["flag"] != true { + t.Fatalf("BooleanProperties flag = %#v, want true", m.ApplicationProperties["flag"]) + } + if m.ApplicationProperties["region"] != "eu" { + t.Fatalf("StringProperties region = %#v, want eu", m.ApplicationProperties["region"]) + } + if m.ApplicationProperties["injected"] != "x" { + t.Fatalf("extra property not merged: %v", m.ApplicationProperties) + } + + // Defaults: absent durable -> true, absent priority -> 4. + m2 := msgs[1] + if m2.Header == nil || !m2.Header.Durable || m2.Header.Priority != defaultPriority { + t.Fatalf("second msg header = %+v, want durable priority %d", m2.Header, defaultPriority) + } +} + +func TestParseArtemisMessagesExtraOverridesFile(t *testing.T) { + data := []byte(`[{ "text": "x", "StringProperties": { "region": "eu" } }]`) + msgs, err := ParseArtemisMessages(data, map[string]interface{}{"region": "override"}) + if err != nil { + t.Fatalf("parse: %v", err) + } + if msgs[0].ApplicationProperties["region"] != "override" { + t.Fatalf("extra property did not override file property: %v", msgs[0].ApplicationProperties) + } +} + +func TestProduceThenBrowseIntegration(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer c.Close(ctx) + + msgs := GenerateMessages(5, 128, map[string]interface{}{"batch": "test"}) + sent, err := c.Produce(ctx, "produce-test", msgs, 0, 1, nil) + if err != nil { + t.Fatalf("produce: %v", err) + } + if sent != 5 { + t.Fatalf("sent = %d, want 5", sent) + } + + // Non-destructive browse must see all 5 with the expected body size. + browsed := browseWithRetry(ctx, t, c, "produce-test", 20, 0) + if len(browsed) != 5 { + t.Fatalf("browsed %d messages, want 5", len(browsed)) + } + for _, b := range browsed { + if b.Size != 128 { + t.Fatalf("browsed message size = %d, want 128", b.Size) + } + } +} + +// TestProduceRateLimited drives the throttled paths in both produceSequential +// (interval>0) and produceParallel (shared ticker), which the other produce +// tests skip by sending at rate 0. +func TestProduceRateLimited(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer c.Close(ctx) + + // A progress counter exercises the onProgress branch on both the sequential + // and parallel paths (the other produce tests pass nil). + var seqProg atomic.Int64 + // rate 500/s keeps the test near-instant while still exercising the + // throttle branch (interval = 2ms between sends). + seqSent, err := c.Produce(ctx, "rate-seq", GenerateMessages(6, 32, nil), 500, 1, + func(int) { seqProg.Add(1) }) + if err != nil { + t.Fatalf("sequential rate produce: %v", err) + } + if seqSent != 6 || seqProg.Load() != 6 { + t.Fatalf("sequential sent = %d, progress = %d, want 6/6", seqSent, seqProg.Load()) + } + + var parProg atomic.Int64 + parSent, err := c.Produce(ctx, "rate-par", GenerateMessages(8, 32, nil), 500, 3, + func(int) { parProg.Add(1) }) + if err != nil { + t.Fatalf("parallel rate produce: %v", err) + } + if parSent != 8 || parProg.Load() != 8 { + t.Fatalf("parallel sent = %d, progress = %d, want 8/8", parSent, parProg.Load()) + } + + // workers > message count must clamp to len(msgs) rather than spin up idle + // senders (or divide by zero). + clampSent, err := c.Produce(ctx, "rate-clamp", GenerateMessages(2, 32, nil), 0, 8, nil) + if err != nil { + t.Fatalf("clamped parallel produce: %v", err) + } + if clampSent != 2 { + t.Fatalf("clamped sent = %d, want 2", clampSent) + } +} + +// TestProduceCancellation covers the mid-flight cancellation paths of both +// produce strategies: a rate limit keeps the send loop running long enough that +// canceling the context lands inside it, so the throttle-cancel and ctx.Err +// returns fire and the call reports the partial count with context.Canceled. +func TestProduceCancellation(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer c.Close(ctx) + + for _, workers := range []int{1, 4} { + // rate 100/s (10ms/msg) over 200 msgs => ~2s of work; cancel after 100ms. + cctx, ccancel := context.WithCancel(ctx) + time.AfterFunc(100*time.Millisecond, ccancel) + sent, err := c.Produce(cctx, "cancel-test", GenerateMessages(200, 32, nil), 100, workers, nil) + ccancel() + if err == nil { + t.Fatalf("workers=%d: want cancellation error, got nil (sent %d)", workers, sent) + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("workers=%d: want context.Canceled, got %v", workers, err) + } + if sent >= 200 { + t.Fatalf("workers=%d: expected partial send before cancel, got %d", workers, sent) + } + } +} + +func TestProduceParallelIntegration(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer c.Close(ctx) + + msgs := GenerateMessages(50, 128, nil) + sent, err := c.Produce(ctx, "produce-parallel-test", msgs, 0, 4, nil) + if err != nil { + t.Fatalf("produce: %v", err) + } + if sent != 50 { + t.Fatalf("sent = %d, want 50", sent) + } + + // Exactly 50 on the queue — parallel workers must not lose or duplicate. + browsed := browseWithRetry(ctx, t, c, "produce-parallel-test", 100, 0) + if len(browsed) != 50 { + t.Fatalf("browsed %d messages, want 50", len(browsed)) + } + for _, b := range browsed { + if b.Size != 128 { + t.Fatalf("browsed message size = %d, want 128", b.Size) + } + } +} diff --git a/internal/broker/redeliver.go b/internal/broker/redeliver.go new file mode 100644 index 0000000..a553e74 --- /dev/null +++ b/internal/broker/redeliver.go @@ -0,0 +1,128 @@ +package broker + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "io" + + "github.com/Azure/go-amqp" + "github.com/martikan/artemisctl/internal/store" +) + +// RedeliverOpts controls how Redeliver replays a store file back to the +// broker. QueueOverride, when set, redirects every record to that queue +// instead of its originally-drained queue. +type RedeliverOpts struct { + QueueOverride string +} + +// Redeliver replays the records in the store file at storePath back onto +// the broker, resuming from the last saved checkpoint. Each record is sent +// to its original queue (or opts.QueueOverride, if set) with its record +// UUID set as the AMQP application property "_AMQ_DUPL_ID" so Artemis's +// duplicate-detection can drop repeats if the same record is redelivered +// more than once (e.g. after a lost/rolled-back checkpoint). The checkpoint +// is only advanced after a send succeeds, so a crash mid-replay leaves the +// checkpoint pointing at the last durably-redelivered record, not beyond +// it. +func (c *Client) Redeliver(ctx context.Context, storePath string, opts RedeliverOpts, onProgress func(sent int)) (sent, coreSkipped int, err error) { + startOffset, err := store.LoadCheckpoint(storePath) + if err != nil { + return 0, 0, fmt.Errorf("load checkpoint: %w", err) + } + r, err := store.OpenReader(storePath) + if err != nil { + return 0, 0, err + } + defer r.Close() + if err := r.SeekTo(startOffset); err != nil { + return 0, 0, err + } + + senders := map[string]*amqp.Sender{} + defer func() { + for _, s := range senders { + _ = s.Close(context.Background()) + } + }() + senderFor := func(queue string) (*amqp.Sender, error) { + if s, ok := senders[queue]; ok { + return s, nil + } + // TargetCapabilities: []string{"queue"} tells Artemis to route this + // as an anycast queue rather than the default multicast address; + // without it the message is routed multicast and never lands in + // the anycast queue, so redelivery would silently fail to restore + // messages. + s, err := c.sess.NewSender(ctx, queue, &amqp.SenderOptions{TargetCapabilities: []string{"queue"}}) + if err != nil { + return nil, fmt.Errorf("sender for %s: %w", queue, err) + } + senders[queue] = s + return s, nil + } + + for { + // Graceful cancellation (e.g. SIGINT): stop between records and + // return a clean context error. The last successful send has already + // advanced the durable checkpoint, so re-running resumes exactly. + if cerr := ctx.Err(); cerr != nil { + return sent, coreSkipped, cerr + } + rec, afterOffset, nerr := r.Next() + if errors.Is(nerr, io.EOF) { + break + } + if nerr != nil { + return sent, coreSkipped, nerr // ErrCorrupt: stop, good prefix already delivered + } + + var msg amqp.Message + switch rec.Kind { + case store.KindCore: + // Core records have no AMQP wire form: convert on send. A record + // that fails to convert is skipped (with the checkpoint advanced so + // it isn't retried forever) rather than aborting the whole replay -- + // the faithful Core record stays in the store for a later attempt. + converted, cerr := coreToAMQP(rec.CorePayload) + if cerr != nil { + coreSkipped++ + if serr := store.SaveCheckpoint(storePath, afterOffset); serr != nil { + return sent, coreSkipped, fmt.Errorf("save checkpoint: %w", serr) + } + continue + } + msg = *converted + default: + if uerr := msg.UnmarshalBinary(rec.AMQP); uerr != nil { + return sent, coreSkipped, fmt.Errorf("unmarshal record: %w", uerr) + } + } + if msg.ApplicationProperties == nil { + msg.ApplicationProperties = map[string]interface{}{} + } + msg.ApplicationProperties["_AMQ_DUPL_ID"] = hex.EncodeToString(rec.UUID[:]) + + queue := rec.Queue + if opts.QueueOverride != "" { + queue = opts.QueueOverride + } + sender, serr := senderFor(queue) + if serr != nil { + return sent, coreSkipped, serr + } + if serr := sender.Send(ctx, &msg, nil); serr != nil { + return sent, coreSkipped, fmt.Errorf("send to %s: %w", queue, serr) + } + if serr := store.SaveCheckpoint(storePath, afterOffset); serr != nil { + return sent, coreSkipped, fmt.Errorf("save checkpoint: %w", serr) + } + sent++ + if onProgress != nil { + onProgress(sent) + } + } + return sent, coreSkipped, nil +} diff --git a/internal/broker/redeliver_core_integration_test.go b/internal/broker/redeliver_core_integration_test.go new file mode 100644 index 0000000..35040b0 --- /dev/null +++ b/internal/broker/redeliver_core_integration_test.go @@ -0,0 +1,92 @@ +// internal/broker/redeliver_core_integration_test.go +package broker + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/Azure/go-amqp" + "github.com/martikan/artemisctl/internal/journal" + "github.com/martikan/artemisctl/internal/store" +) + +// TestRedeliverCoreConvertsAndLands writes a KindCore store record (a decoded +// Core TEXT message), redelivers it, and confirms the converted AMQP message +// actually lands on the queue with the right body and application property — +// the end-to-end proof that Core messages salvaged offline can be replayed to +// a live broker via Core->AMQP conversion. +func TestRedeliverCoreConvertsAndLands(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx) + + const queue = "coreq" + payload := (&journal.CorePayload{ + MessageID: 1, + Address: queue, + Type: journal.CoreTypeText, + Durable: true, + Priority: 4, + Properties: map[string]any{"region": "eu"}, + Body: coreSimpleString("core-hello"), + }).Encode() + + path := filepath.Join(t.TempDir(), "core.artx") + w, err := store.NewWriter(path) + if err != nil { + t.Fatal(err) + } + rec := store.Record{ + UUID: store.DedupIDCore(payload, queue), + Queue: queue, + DrainedAt: time.Now().UnixNano(), + Kind: store.KindCore, + CorePayload: payload, + } + if err := w.Append(rec); err != nil { + t.Fatal(err) + } + if err := w.Sync(); err != nil { + t.Fatal(err) + } + _ = w.Close() + + n, coreSkipped, err := c.Redeliver(ctx, path, RedeliverOpts{}, nil) + if err != nil { + t.Fatalf("redeliver: %v", err) + } + if n != 1 || coreSkipped != 0 { + t.Fatalf("redeliver core: sent=%d skipped=%d, want 1/0", n, coreSkipped) + } + + recv, err := c.Session().NewReceiver(ctx, queue, &amqp.ReceiverOptions{SourceCapabilities: []string{"queue"}}) + if err != nil { + t.Fatalf("open receiver: %v", err) + } + defer recv.Close(context.Background()) + rctx, rcancel := context.WithTimeout(ctx, 10*time.Second) + defer rcancel() + msg, err := recv.Receive(rctx, nil) + if err != nil { + t.Fatalf("receive: %v", err) + } + _ = recv.AcceptMessage(ctx, msg) + + if got, ok := msg.Value.(string); !ok || got != "core-hello" { + t.Errorf("body = %#v, want AmqpValue \"core-hello\"", msg.Value) + } + if got := msg.ApplicationProperties["region"]; got != "eu" { + t.Errorf("app property region = %v, want eu", got) + } +} diff --git a/internal/broker/redeliver_integration_test.go b/internal/broker/redeliver_integration_test.go new file mode 100644 index 0000000..99a61b5 --- /dev/null +++ b/internal/broker/redeliver_integration_test.go @@ -0,0 +1,133 @@ +// internal/broker/redeliver_integration_test.go +package broker + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" + + "github.com/martikan/artemisctl/internal/store" +) + +func TestRedeliverRoundTripAndDedup(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx) + + if err := sendTestMessages(ctx, c, "orders", []string{"m1", "m2", "m3"}); err != nil { + t.Fatalf("send test messages: %v", err) + } + + // Drain to a store file. + path := filepath.Join(t.TempDir(), "dump.artx") + w, err := store.NewWriter(path) + if err != nil { + t.Fatal(err) + } + if _, err := c.DrainQueue(ctx, "orders", w, 3*time.Second, 10); err != nil { + t.Fatal(err) + } + _ = w.Close() + + // Redeliver back. + n, _, err := c.Redeliver(ctx, path, RedeliverOpts{}, nil) + if err != nil { + t.Fatalf("redeliver: %v", err) + } + if n != 3 { + t.Fatalf("want 3 redelivered, got %d", n) + } + + // Redeliver AGAIN from offset 0 (simulate lost checkpoint) -> dedup should drop repeats. + if err := store.SaveCheckpoint(path, 0); err != nil { + t.Fatal(err) + } + if _, _, err := c.Redeliver(ctx, path, RedeliverOpts{}, nil); err != nil { + t.Fatal(err) + } + + // Queue should hold exactly 3 (dedup prevented 6). + stats, err := c.ListQueues(ctx) + if err != nil { + t.Fatal(err) + } + var count int64 = -1 + for _, s := range stats { + if s.Name == "orders" { + count = s.MessageCount + } + } + if count != 3 { + t.Fatalf("dedup failed: orders has %d messages, want 3", count) + } +} + +// TestRedeliverGracefulCancel models the I3 SIGINT path with a context cancel +// (real signals are flaky): cancelling mid-replay must return promptly with a +// context error, no panic, and a durable checkpoint at the last successfully +// sent record so the same command resumes exactly. +func TestRedeliverGracefulCancel(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemis(t) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + c, err := Connect(ctx, props) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx) + + if err := sendTestMessages(ctx, c, "cancelq", []string{"c1", "c2", "c3", "c4", "c5"}); err != nil { + t.Fatalf("seed: %v", err) + } + path := filepath.Join(t.TempDir(), "dump.artx") + w, err := store.NewWriter(path) + if err != nil { + t.Fatal(err) + } + if _, err := c.DrainQueue(ctx, "cancelq", w, 3*time.Second, 10); err != nil { + t.Fatal(err) + } + _ = w.Close() + + // Cancel the replay after the first record is sent + checkpointed. The + // next Send observes the canceled context and Redeliver returns gracefully. + replayCtx, replayCancel := context.WithCancel(ctx) + defer replayCancel() + n, _, err := c.Redeliver(replayCtx, path, RedeliverOpts{}, func(sent int) { + if sent == 1 { + replayCancel() + } + }) + if err == nil { + t.Fatalf("want a context error after cancel, got nil (n=%d)", n) + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("want context.Canceled, got: %v", err) + } + if n < 1 || n >= 5 { + t.Fatalf("want a partial redeliver (1..4), got %d", n) + } + // Checkpoint must be durable and point past the delivered records. + off, err := store.LoadCheckpoint(path) + if err != nil { + t.Fatalf("load checkpoint: %v", err) + } + if off <= 0 { + t.Fatalf("checkpoint not saved on graceful cancel: offset=%d", off) + } +} diff --git a/internal/broker/redeliver_test.go b/internal/broker/redeliver_test.go new file mode 100644 index 0000000..4cf137c --- /dev/null +++ b/internal/broker/redeliver_test.go @@ -0,0 +1,91 @@ +// internal/broker/redeliver_test.go +package broker + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/martikan/artemisctl/internal/store" +) + +// TestRedeliverStopsOnCorruptRecord proves that Redeliver detects a corrupt +// first record and returns immediately, WITHOUT ever touching the broker +// session (senderFor / c.sess.NewSender is only reached after a successful +// r.Next()). This is exercised against a zero-value &Client{} whose sess +// field is nil: if Redeliver tried to dereference c.sess before returning +// the ErrCorrupt error, this test would panic instead of failing cleanly. +func TestRedeliverStopsOnCorruptRecord(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "dump.artx") + + w, err := store.NewWriter(path) + if err != nil { + t.Fatalf("new writer: %v", err) + } + rec := store.Record{ + UUID: [16]byte{1, 2, 3, 4}, + Queue: "orders", + DrainedAt: 1234, + AMQP: []byte("some amqp payload bytes"), + } + if err := w.Append(rec); err != nil { + t.Fatalf("append: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close writer: %v", err) + } + + corruptRecordBody(t, path) + + c := &Client{} // zero-value: sess is nil + + var n int + var redelivErr error + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Redeliver panicked: %v", r) + } + }() + n, _, redelivErr = c.Redeliver(context.Background(), path, RedeliverOpts{}, nil) + }() + + if n != 0 { + t.Fatalf("want count 0, got %d", n) + } + if redelivErr == nil { + t.Fatal("want non-nil error, got nil") + } + if !errors.Is(redelivErr, store.ErrCorrupt) { + t.Fatalf("want errors.Is(err, store.ErrCorrupt), got: %v", redelivErr) + } +} + +// corruptRecordBody flips a byte inside the body of the first record in the +// store file at path, so its CRC no longer matches and store.OpenReader(...) +// .Next() returns store.ErrCorrupt on the very first read. +// +// Layout: 5-byte header (magic + version) | 4-byte body length | body | 4-byte +// crc32. We flip a byte a few bytes into the body (well past the UUID/offset +// fields that only affect metadata) so the CRC check fails deterministically. +func corruptRecordBody(t *testing.T, path string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read store file: %v", err) + } + const headerLen = 5 + const lenPrefixLen = 4 + bodyStart := headerLen + lenPrefixLen + if len(data) <= bodyStart+10 { + t.Fatalf("store file too small to corrupt: %d bytes", len(data)) + } + idx := bodyStart + 10 // a handful of bytes into the body + data[idx] ^= 0xFF + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("rewrite store file: %v", err) + } +} diff --git a/internal/brokertest/broker.go b/internal/brokertest/broker.go new file mode 100644 index 0000000..1d1cc7d --- /dev/null +++ b/internal/brokertest/broker.go @@ -0,0 +1,91 @@ +// Package brokertest provides a single shared Artemis broker container reused +// by every integration test across packages. It lives in a normal (non-_test) +// package because test-only symbols cannot be shared across package boundaries. +package brokertest + +import ( + "context" + "testing" + "time" + + tc "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// Conn holds the address and credentials of the shared broker. It is a plain +// struct rather than broker.ConnectionProps so this package does not import +// internal/broker: an internal (package broker) test importing a package that +// imports broker would be an import cycle. Callers map it to ConnectionProps. +type Conn struct { + URL string + Username string + Password string +} + +const ( + // sharedName is the fixed container name that makes Reuse attach to one + // instance across test binaries instead of starting a new one each time. + sharedName = "artemisctl-it-broker" + + // sharedImage is recent enough to expose the two-arg JSON + // addAddressSettings(String,String) overload cordon relies on (2.33+) and + // boots reliably under the NIO journal. + sharedImage = "apache/activemq-artemis:2.42.0-alpine" + + sharedUser = "artemis" + sharedPass = "artemis" +) + +// PermissiveWildcardSettings is an address-settings JSON for the wildcard match +// that disables the address-full limits, used to lift any leftover cordon at the +// start of each test. Applying this via Client.Uncordon reliably overrides a +// cordon's FAIL policy — unlike removeAddressSettings("#"), which reports success +// but does not actually clear the wildcard override on Artemis 2.42. The DLA, +// expiry, and auto-create fields match the broker defaults so tests that rely on +// DLQ/ExpiryQueue and queue auto-creation keep working. +const PermissiveWildcardSettings = `{"addressFullMessagePolicy":"PAGE","maxSizeBytes":104857600,"maxReadPageBytes":20971520,"maxReadPageMessages":-1,"pageLimitBytes":-1,"pageLimitMessages":-1,"maxSizeMessages":-1,"pageSizeBytes":10485760,"messageCounterHistoryDayLimit":10,"redeliveryDelay":0,"deadLetterAddress":"DLQ","expiryAddress":"ExpiryQueue","slowConsumerThresholdMeasurementUnit":"MESSAGES_PER_SECOND","autoCreateQueues":true,"autoDeleteQueues":false,"autoCreateAddresses":true,"autoDeleteAddresses":false,"managementBrowsePageSize":200,"maxSizeBytesRejectThreshold":-1}` + +// Shared starts, or attaches to, the single reused Artemis broker used by all +// integration tests and returns connection props for it. +// +// Dirty context: the container is deliberately NOT terminated. Reused +// containers are excluded from the Ryuk reaper, so the broker persists across +// tests and test binaries; the first caller starts it and every later caller +// (in either package) attaches by name. Because state accumulates, tests must +// use distinct queue/address names, and cordon tests must restore their +// settings so the broker is left usable for the next test. +func Shared(t testing.TB) Conn { + t.Helper() + ctx := context.Background() + req := tc.ContainerRequest{ + Name: sharedName, + Image: sharedImage, + ExposedPorts: []string{"61616/tcp"}, + Env: map[string]string{ + "ARTEMIS_USER": sharedUser, + "ARTEMIS_PASSWORD": sharedPass, + // --nio: newer images default to the AIO journal, which fails + // io_getevents under rootless container runtimes; NIO boots + // everywhere. --relax-jolokia keeps the management console reachable. + "EXTRA_ARGS": "--nio --relax-jolokia", + }, + WaitingFor: wait.ForListeningPort("61616/tcp").WithStartupTimeout(120 * time.Second), + } + ctr, err := tc.GenericContainer(ctx, tc.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + Reuse: true, + }) + if err != nil { + t.Fatalf("start shared artemis: %v", err) + } + host, err := ctr.Host(ctx) + if err != nil { + t.Fatalf("host: %v", err) + } + port, err := ctr.MappedPort(ctx, "61616/tcp") + if err != nil { + t.Fatalf("port: %v", err) + } + return Conn{URL: host + ":" + port.Port(), Username: sharedUser, Password: sharedPass} +} diff --git a/internal/brokertest/broker_test.go b/internal/brokertest/broker_test.go new file mode 100644 index 0000000..b77433d --- /dev/null +++ b/internal/brokertest/broker_test.go @@ -0,0 +1,19 @@ +package brokertest + +import "testing" + +// TestSharedBoots smoke-tests the shared broker helper: it returns a usable +// endpoint and credentials (and, via Reuse, attaches to the one shared container +// the rest of the integration suite uses). +func TestSharedBoots(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + c := Shared(t) + if c.URL == "" { + t.Fatal("Shared returned empty URL") + } + if c.Username == "" || c.Password == "" { + t.Fatalf("Shared returned empty credentials: %+v", c) + } +} diff --git a/internal/cli/browse.go b/internal/cli/browse.go new file mode 100644 index 0000000..988b37a --- /dev/null +++ b/internal/cli/browse.go @@ -0,0 +1,69 @@ +package cli + +import ( + "context" + "fmt" + "sort" + "text/tabwriter" + + "github.com/martikan/artemisctl/internal/broker" + "github.com/spf13/cobra" +) + +func newBrowseCmd() *cobra.Command { + var queue, message string + var limit, offset int + + cmd := &cobra.Command{ + Use: "browse", + Short: "Non-destructively peek at messages in a queue", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx, cancel := connectCtx(cmd, context.Background()) + defer cancel() + c, err := broker.Connect(ctx, connProps(cmd)) + if err != nil { + return err + } + defer c.Close(ctx) + + if message != "" { + m, err := c.BrowseMessage(ctx, queue, message) + if err != nil { + return err + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "Message %s on %s\n", message, queue) + fmt.Fprintf(out, "Body: %s\n", string(m.GetData())) + if len(m.ApplicationProperties) > 0 { + fmt.Fprintln(out, "Properties:") + keys := make([]string, 0, len(m.ApplicationProperties)) + for k := range m.ApplicationProperties { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(out, " %s = %v\n", k, m.ApplicationProperties[k]) + } + } + return nil + } + + msgs, err := c.BrowseQueue(ctx, queue, limit, offset) + if err != nil { + return err + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "ID\tSIZE\tTIMESTAMP\tPREVIEW") + for _, m := range msgs { + fmt.Fprintf(w, "%s\t%d\t%d\t%s\n", m.ID, m.Size, m.Timestamp, m.Preview) + } + return w.Flush() + }, + } + cmd.Flags().StringVar(&queue, "queue", "", "queue to browse (required)") + cmd.Flags().StringVar(&message, "message", "", "show full body for a single message ID") + cmd.Flags().IntVar(&limit, "limit", 20, "max messages to list") + cmd.Flags().IntVar(&offset, "offset", 0, "skip the first N messages") + _ = cmd.MarkFlagRequired("queue") + return cmd +} diff --git a/internal/cli/commands_integration_test.go b/internal/cli/commands_integration_test.go new file mode 100644 index 0000000..1151e80 --- /dev/null +++ b/internal/cli/commands_integration_test.go @@ -0,0 +1,143 @@ +// internal/cli/commands_integration_test.go +package cli + +import ( + "bytes" + "path/filepath" + "strings" + "testing" +) + +// TestCommandsAgainstBroker exercises every connect-requiring command's success +// path against one shared broker container: status, health, browse, then an +// export→redeliver round-trip. Bundling them into a single container boot keeps +// the integration suite fast while covering the RunE bodies the unit tests stop +// short of (they only reach the Connect error). +func TestCommandsAgainstBroker(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemisForCLI(t) + seedQueue(t, props, "orders", []string{"alpha", "bravo", "charlie"}) + + run := func(t *testing.T, args ...string) (string, error) { + t.Helper() + root := NewRootCmd() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + conn := []string{"--url", props.URL, "-u", props.Username, "-p", props.Password} + root.SetArgs(append(args, conn...)) + err := root.Execute() + return out.String(), err + } + + t.Run("status", func(t *testing.T) { + out, err := run(t, "status") + if err != nil { + t.Fatalf("status: %v", err) + } + if !strings.Contains(out, "orders") { + t.Fatalf("status output missing queue: %q", out) + } + }) + + t.Run("health", func(t *testing.T) { + out, err := run(t, "health") + if err != nil { + t.Fatalf("health: %v", err) + } + if !strings.Contains(out, "verdict:") { + t.Fatalf("health output missing verdict: %q", out) + } + }) + + t.Run("browse", func(t *testing.T) { + out, err := run(t, "browse", "--queue", "orders", "--limit", "10") + if err != nil { + t.Fatalf("browse: %v", err) + } + for _, want := range []string{"alpha", "bravo", "charlie"} { + if !strings.Contains(out, want) { + t.Fatalf("browse output missing %q: %q", want, out) + } + } + + // The single-message path: pull an ID from the list output's first data + // row and fetch its full body via --message. + id := firstMessageID(t, out) + body, err := run(t, "browse", "--queue", "orders", "--message", id) + if err != nil { + t.Fatalf("browse --message: %v", err) + } + if !strings.Contains(body, "Body:") { + t.Fatalf("browse --message output missing body: %q", body) + } + }) + + // 100+ messages so the export→redeliver round-trip drains enough records to + // fire redeliver's every-100 progress callback. Tagged with a property so the + // browse --message path below exercises the property-printing block. + t.Run("produce", func(t *testing.T) { + out, err := run(t, "produce", "--queue", "generated", "--count", "100", "--size", "64", + "--property", "batch=x") + if err != nil { + t.Fatalf("produce: %v", err) + } + if !strings.Contains(out, "produced 100 messages") { + t.Fatalf("produce output unexpected: %q", out) + } + }) + + t.Run("browse message with properties", func(t *testing.T) { + list, err := run(t, "browse", "--queue", "generated", "--limit", "1") + if err != nil { + t.Fatalf("browse generated: %v", err) + } + id := firstMessageID(t, list) + body, err := run(t, "browse", "--queue", "generated", "--message", id) + if err != nil { + t.Fatalf("browse --message: %v", err) + } + if !strings.Contains(body, "Properties:") || !strings.Contains(body, "batch = x") { + t.Fatalf("browse --message output missing properties: %q", body) + } + }) + + // Round-trip: export drains orders to a store file, redeliver replays it. + store := filepath.Join(t.TempDir(), "dump.artx") + t.Run("export", func(t *testing.T) { + out, err := run(t, "export", "--out", store, "--drain-timeout", "3s") + if err != nil { + t.Fatalf("export: %v", err) + } + if !strings.Contains(out, "exported") { + t.Fatalf("export output unexpected: %q", out) + } + }) + + t.Run("redeliver", func(t *testing.T) { + out, err := run(t, "redeliver", "--in", store) + if err != nil { + t.Fatalf("redeliver: %v", err) + } + if !strings.Contains(out, "redelivered") { + t.Fatalf("redeliver output unexpected: %q", out) + } + }) +} + +// firstMessageID extracts the ID from the first data row of `browse` list +// output (tab/space-aligned: ID SIZE TIMESTAMP PREVIEW, with a header row). +func firstMessageID(t *testing.T, listOutput string) string { + t.Helper() + for _, line := range strings.Split(listOutput, "\n") { + fields := strings.Fields(line) + if len(fields) == 0 || fields[0] == "ID" { + continue // blank or header row + } + return fields[0] + } + t.Fatalf("no message ID found in browse output: %q", listOutput) + return "" +} diff --git a/internal/cli/commands_test.go b/internal/cli/commands_test.go new file mode 100644 index 0000000..4430c15 --- /dev/null +++ b/internal/cli/commands_test.go @@ -0,0 +1,195 @@ +package cli + +import ( + "bytes" + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/spf13/cobra" +) + +// runCmd executes the root command with args, capturing combined output. +func runCmd(t *testing.T, args ...string) (string, error) { + t.Helper() + root := NewRootCmd() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs(args) + err := root.Execute() + return out.String(), err +} + +// unreachable points at a port nothing listens on, so broker.Connect fails +// fast with connection-refused; it lets the tests drive each command's RunE up +// to (and through) the Connect error path without a live broker. The short +// --timeout bounds the dial in case the OS is slow to refuse. +var unreachable = []string{"--url", "127.0.0.1:1", "--timeout", "3s"} + +func TestSubcommandsRegistered(t *testing.T) { + root := NewRootCmd() + want := []string{"status", "health", "browse", "produce", "export", "redeliver"} + have := map[string]bool{} + for _, c := range root.Commands() { + have[c.Name()] = true + } + for _, w := range want { + if !have[w] { + t.Errorf("subcommand %q not registered", w) + } + } +} + +func TestRequiredFlagsEnforced(t *testing.T) { + cases := []struct { + name string + args []string + flag string + }{ + {"browse", []string{"browse"}, "queue"}, + {"produce", []string{"produce"}, "queue"}, + {"redeliver", []string{"redeliver"}, "in"}, + {"export", []string{"export"}, "out"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := runCmd(t, tc.args...) + if err == nil || !strings.Contains(err.Error(), tc.flag) { + t.Fatalf("want required-flag error mentioning %q, got %v", tc.flag, err) + } + }) + } +} + +func TestParseProperties(t *testing.T) { + got, err := parseProperties(nil) + if err != nil || got != nil { + t.Fatalf("nil input: got %v, %v; want nil, nil", got, err) + } + + got, err = parseProperties([]string{"region=eu", "tier=gold"}) + if err != nil { + t.Fatalf("valid: %v", err) + } + if got["region"] != "eu" || got["tier"] != "gold" { + t.Fatalf("parsed = %v", got) + } + + for _, bad := range []string{"novalue", "=orphan"} { + if _, err := parseProperties([]string{bad}); err == nil { + t.Fatalf("parseProperties(%q) should error", bad) + } + } +} + +// TestProduceNoMessages: --count 0 generates nothing and short-circuits before +// any broker connection, so it must succeed and say so. +func TestProduceNoMessages(t *testing.T) { + out, err := runCmd(t, "produce", "--queue", "q", "--count", "0") + if err != nil { + t.Fatalf("produce --count 0: %v", err) + } + if !strings.Contains(out, "no messages to send") { + t.Fatalf("output = %q, want no-messages notice", out) + } +} + +func TestProduceInvalidProperty(t *testing.T) { + _, err := runCmd(t, "produce", "--queue", "q", "--property", "novalue") + if err == nil || !strings.Contains(err.Error(), "--property") { + t.Fatalf("want --property error, got %v", err) + } +} + +func TestProduceMissingFile(t *testing.T) { + _, err := runCmd(t, "produce", "--queue", "q", "--file", "/no/such/file.json") + if err == nil || !strings.Contains(err.Error(), "read message file") { + t.Fatalf("want read-file error, got %v", err) + } +} + +// TestConnectFailurePaths drives each connect-requiring command against an +// unreachable broker; every one must surface a non-nil error rather than hang +// or panic. +func TestConnectFailurePaths(t *testing.T) { + tmp := filepath.Join(t.TempDir(), "store.artx") + cases := map[string][]string{ + "status": {"status"}, + "health": {"health"}, + "browse": {"browse", "--queue", "q"}, + "produce": {"produce", "--queue", "q", "--count", "1"}, + "export": {"export", "--out", tmp}, + "redeliver": {"redeliver", "--in", tmp}, + } + for name, base := range cases { + t.Run(name, func(t *testing.T) { + _, err := runCmd(t, append(base, unreachable...)...) + if err == nil { + t.Fatalf("%s against unreachable broker: want error, got nil", name) + } + }) + } +} + +func TestResolvePassword(t *testing.T) { + t.Setenv("ARTEMIS_PASSWORD", "") + if got := resolvePassword("flagpw"); got != "flagpw" { + t.Fatalf("empty env: got %q, want flagpw", got) + } + t.Setenv("ARTEMIS_PASSWORD", "envpw") + if got := resolvePassword("flagpw"); got != "envpw" { + t.Fatalf("env set: got %q, want envpw", got) + } +} + +func TestConnectCtxHonorsTimeout(t *testing.T) { + cmd := &cobra.Command{} + cmd.Flags().Duration("timeout", 0, "") + + // timeout > 0 -> deadline is set. + _ = cmd.Flags().Set("timeout", "5s") + ctx, cancel := connectCtx(cmd, context.Background()) + defer cancel() + if _, ok := ctx.Deadline(); !ok { + t.Fatal("timeout=5s: expected a deadline") + } + + // timeout <= 0 -> cancelable, but no deadline. + _ = cmd.Flags().Set("timeout", "0s") + ctx2, cancel2 := connectCtx(cmd, context.Background()) + defer cancel2() + if _, ok := ctx2.Deadline(); ok { + t.Fatal("timeout=0: expected no deadline") + } +} + +func TestConnPropsReadsFlags(t *testing.T) { + root := NewRootCmd() + root.SetArgs([]string{"status", "--url", "h:1", "-u", "bob", "-p", "secret"}) + // Parse flags without executing RunE by resolving the target command. + target, _, err := root.Find([]string{"status", "--url", "h:1", "-u", "bob", "-p", "secret"}) + if err != nil { + t.Fatal(err) + } + if err := target.ParseFlags([]string{"--url", "h:1", "-u", "bob", "-p", "secret"}); err != nil { + t.Fatal(err) + } + t.Setenv("ARTEMIS_PASSWORD", "") + p := connProps(target) + if p.URL != "h:1" || p.Username != "bob" || p.Password != "secret" { + t.Fatalf("connProps = %+v", p) + } +} + +func TestSignalCtxCancelable(t *testing.T) { + ctx, stop := signalCtx() + defer stop() + select { + case <-ctx.Done(): + t.Fatal("signalCtx already canceled") + case <-time.After(10 * time.Millisecond): + } +} diff --git a/internal/cli/cordon.go b/internal/cli/cordon.go new file mode 100644 index 0000000..ccd75a7 --- /dev/null +++ b/internal/cli/cordon.go @@ -0,0 +1,115 @@ +package cli + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/martikan/artemisctl/internal/broker" + "github.com/spf13/cobra" +) + +// defaultCordonState is where cordon stashes the pre-cordon settings so a later +// uncordon (a separate process) can restore them. +const defaultCordonState = "artemisctl-cordon.json" + +// cordonState is the sidecar written by cordon and consumed by uncordon. It +// persists the pre-cordon wildcard address-settings so the block can be lifted +// faithfully from a different invocation. +type cordonState struct { + BrokerURL string `json:"brokerURL"` + SavedSettings string `json:"savedSettings"` + CordonedAt time.Time `json:"cordonedAt"` +} + +func writeCordonState(path string, s cordonState) error { + b, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, b, 0o600) +} + +func readCordonState(path string) (cordonState, error) { + var s cordonState + b, err := os.ReadFile(path) + if err != nil { + return s, err + } + err = json.Unmarshal(b, &s) + return s, err +} + +func newCordonCmd() *cobra.Command { + var stateFile string + var yes bool + + cmd := &cobra.Command{ + Use: "cordon", + Short: "Block producers broker-wide (address-full BLOCK) before an export", + Long: "cordon applies an address-full BLOCK policy to every address so no new " + + "messages can be produced, while consumers and export keep working. The " + + "pre-cordon settings are saved to a state file for uncordon to restore. " + + "Requires a broker new enough to accept addAddressSettings over AMQP (2.33+).", + RunE: func(cmd *cobra.Command, _ []string) error { + if _, err := os.Stat(stateFile); err == nil { + return fmt.Errorf("state file %s already exists; broker may already be cordoned (uncordon first, or remove the file)", stateFile) + } + if !yes && !confirm(cmd, "This blocks ALL producers on the broker. Continue? [y/N] ") { + fmt.Fprintln(cmd.OutOrStdout(), "aborted") + return nil + } + + ctx, cancel := connectCtx(cmd, context.Background()) + defer cancel() + props := connProps(cmd) + c, err := broker.Connect(ctx, props) + if err != nil { + return err + } + defer c.Close(context.Background()) + + saved, err := c.Cordon(ctx) + if err != nil { + if errors.Is(err, broker.ErrBrokerTooOld) { + return fmt.Errorf("%w — cordon over AMQP is unavailable on this broker", err) + } + return err + } + if err := writeCordonState(stateFile, cordonState{ + BrokerURL: props.URL, + SavedSettings: saved, + CordonedAt: time.Now(), + }); err != nil { + return fmt.Errorf("cordon applied but failed to write state file %s: %w", stateFile, err) + } + fmt.Fprintf(cmd.OutOrStdout(), "broker cordoned; producers blocked. state saved to %s\n", stateFile) + fmt.Fprintln(cmd.OutOrStdout(), "run 'artemisctl uncordon' to lift the block") + return nil + }, + } + cmd.Flags().StringVar(&stateFile, "state-file", defaultCordonState, "path to persist pre-cordon settings for uncordon") + cmd.Flags().BoolVar(&yes, "yes", false, "skip the confirmation prompt") + return cmd +} + +// confirm reads a yes/no answer from the command's input stream. +func confirm(cmd *cobra.Command, prompt string) bool { + fmt.Fprint(cmd.OutOrStdout(), prompt) + r := bufio.NewReader(cmd.InOrStdin()) + line, err := r.ReadString('\n') + if err != nil { + return false + } + switch strings.ToLower(strings.TrimSpace(line)) { + case "y", "yes": + return true + default: + return false + } +} diff --git a/internal/cli/cordon_integration_test.go b/internal/cli/cordon_integration_test.go new file mode 100644 index 0000000..e1c9b92 --- /dev/null +++ b/internal/cli/cordon_integration_test.go @@ -0,0 +1,84 @@ +// internal/cli/cordon_integration_test.go +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestCordonUncordonAgainstBroker drives the cordon/uncordon commands end-to-end +// against the shared broker: the happy cordon→restore cycle plus the two +// no-state-file uncordon branches (error without --force-remove, wildcard removal +// with it). This covers the RunE success bodies the unit tests stop short of. +func TestCordonUncordonAgainstBroker(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + props := startArtemisForCLI(t) + + run := func(t *testing.T, args ...string) (string, error) { + t.Helper() + root := NewRootCmd() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + conn := []string{"--url", props.URL, "-u", props.Username, "-p", props.Password} + root.SetArgs(append(args, conn...)) + err := root.Execute() + return out.String(), err + } + + state := filepath.Join(t.TempDir(), "cordon.json") + + t.Run("cordon", func(t *testing.T) { + out, err := run(t, "cordon", "--yes", "--state-file", state) + if err != nil { + t.Fatalf("cordon: %v", err) + } + if !strings.Contains(out, "broker cordoned") { + t.Fatalf("cordon output unexpected: %q", out) + } + s, err := readCordonState(state) + if err != nil { + t.Fatalf("state file not written: %v", err) + } + if s.SavedSettings == "" { + t.Fatal("state file recorded empty saved settings") + } + }) + + t.Run("uncordon restores and removes state", func(t *testing.T) { + out, err := run(t, "uncordon", "--state-file", state) + if err != nil { + t.Fatalf("uncordon: %v", err) + } + if !strings.Contains(out, "settings restored") { + t.Fatalf("uncordon output unexpected: %q", out) + } + if _, statErr := os.Stat(state); !os.IsNotExist(statErr) { + t.Fatal("uncordon should remove the state file after restoring") + } + }) + + t.Run("uncordon without state file errors", func(t *testing.T) { + missing := filepath.Join(t.TempDir(), "none.json") + _, err := run(t, "uncordon", "--state-file", missing) + if err == nil || !strings.Contains(err.Error(), "no state file") { + t.Fatalf("want no-state-file error, got %v", err) + } + }) + + t.Run("uncordon --force-remove without state file", func(t *testing.T) { + missing := filepath.Join(t.TempDir(), "none.json") + out, err := run(t, "uncordon", "--force-remove", "--state-file", missing) + if err != nil { + t.Fatalf("uncordon --force-remove: %v", err) + } + if !strings.Contains(out, "wildcard settings entry removed") { + t.Fatalf("force-remove output unexpected: %q", out) + } + }) +} diff --git a/internal/cli/cordon_test.go b/internal/cli/cordon_test.go new file mode 100644 index 0000000..55a0037 --- /dev/null +++ b/internal/cli/cordon_test.go @@ -0,0 +1,127 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/spf13/cobra" +) + +// runCmdStdin is runCmd with a fixed stdin, so prompts (confirm) can be driven. +func runCmdStdin(t *testing.T, stdin string, args ...string) (string, error) { + t.Helper() + root := NewRootCmd() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetIn(strings.NewReader(stdin)) + root.SetArgs(args) + err := root.Execute() + return out.String(), err +} + +func TestCordonStateRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + want := cordonState{ + BrokerURL: "broker:61616", + SavedSettings: `{"maxSizeBytes":123}`, + CordonedAt: time.Now().Truncate(time.Second), + } + if err := writeCordonState(path, want); err != nil { + t.Fatalf("writeCordonState: %v", err) + } + got, err := readCordonState(path) + if err != nil { + t.Fatalf("readCordonState: %v", err) + } + if got.BrokerURL != want.BrokerURL || got.SavedSettings != want.SavedSettings || !got.CordonedAt.Equal(want.CordonedAt) { + t.Fatalf("round-trip mismatch: got %+v want %+v", got, want) + } +} + +func TestReadCordonStateMissingFile(t *testing.T) { + _, err := readCordonState(filepath.Join(t.TempDir(), "does-not-exist.json")) + if err == nil { + t.Fatal("reading a missing state file should error") + } +} + +func TestConfirm(t *testing.T) { + cases := map[string]bool{ + "y\n": true, + "yes\n": true, + "YES\n": true, + "n\n": false, + "no\n": false, + "": false, // EOF -> false + } + for in, want := range cases { + cmd := &cobra.Command{} + cmd.SetIn(strings.NewReader(in)) + var out bytes.Buffer + cmd.SetOut(&out) + if got := confirm(cmd, "continue? "); got != want { + t.Errorf("confirm(%q) = %v, want %v", in, got, want) + } + if !strings.Contains(out.String(), "continue?") { + t.Errorf("confirm(%q) did not write the prompt", in) + } + } +} + +// TestCordonStateFileExists: cordon refuses to run when the state file is +// already present (broker may already be cordoned), before touching the broker. +func TestCordonStateFileExists(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + if err := os.WriteFile(path, []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + _, err := runCmdStdin(t, "", "cordon", "--state-file", path) + if err == nil || !strings.Contains(err.Error(), "already exists") { + t.Fatalf("want already-exists error, got %v", err) + } +} + +// TestCordonAbort: without --yes and a "no" answer, cordon aborts before +// connecting and leaves no state file behind. +func TestCordonAbort(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + out, err := runCmdStdin(t, "n\n", "cordon", "--state-file", path) + if err != nil { + t.Fatalf("abort should not error: %v", err) + } + if !strings.Contains(out, "aborted") { + t.Fatalf("output = %q, want abort notice", out) + } + if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { + t.Fatal("aborted cordon must not write a state file") + } +} + +// TestCordonConnectFailure: with --yes the prompt is skipped and cordon fails at +// connect against an unreachable broker, writing no state file. +func TestCordonConnectFailure(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + args := append([]string{"cordon", "--yes", "--state-file", path}, unreachable...) + _, err := runCmdStdin(t, "", args...) + if err == nil { + t.Fatal("cordon against unreachable broker: want error, got nil") + } + if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { + t.Fatal("failed cordon must not write a state file") + } +} + +// TestUncordonConnectFailure: uncordon connects before reading state, so an +// unreachable broker surfaces a connect error. +func TestUncordonConnectFailure(t *testing.T) { + args := append([]string{"uncordon"}, unreachable...) + _, err := runCmdStdin(t, "", args...) + if err == nil { + t.Fatal("uncordon against unreachable broker: want error, got nil") + } +} diff --git a/internal/cli/export.go b/internal/cli/export.go new file mode 100644 index 0000000..c3d848a --- /dev/null +++ b/internal/cli/export.go @@ -0,0 +1,77 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/martikan/artemisctl/internal/broker" + "github.com/martikan/artemisctl/internal/store" + "github.com/spf13/cobra" +) + +func newExportCmd() *cobra.Command { + var out string + var drainTimeout time.Duration + var batch int + + cmd := &cobra.Command{ + Use: "export", + Short: "Drain every queue (destructive) into a local store file", + RunE: func(cmd *cobra.Command, _ []string) error { + // Bound Connect by --timeout so an unresponsive broker fails fast. + connCtx, connCancel := connectCtx(cmd, context.Background()) + defer connCancel() + c, err := broker.Connect(connCtx, connProps(cmd)) + if err != nil { + return err + } + // The drain itself runs under a separate long-lived context that + // is NOT capped by --timeout (a large drain may exceed it) but IS + // cancelable via SIGINT for a graceful stop. + ctx, stop := signalCtx() + defer stop() + defer c.Close(context.Background()) + + w, err := store.NewWriter(out) + if err != nil { + return err + } + defer w.Close() + // This is a fresh store (NewWriter refuses a non-empty existing + // file). Remove any stale sidecar checkpoint left by a prior + // redeliver of a same-named store; otherwise a later `redeliver` + // would seek to a bogus offset and silently skip records. + if err := os.Remove(out + ".ckpt"); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove stale checkpoint: %w", err) + } + + total, err := c.DrainAll(ctx, w, drainTimeout, batch, func(q string, n int) { + fmt.Fprintf(cmd.OutOrStdout(), "drained %d from %s\n", n, q) + }) + if err != nil { + // SIGINT: drained records are already fsync'd; report a clean + // interruption rather than a crash. Re-running export needs a + // fresh --out path (an existing non-empty store is refused). + if errors.Is(err, context.Canceled) { + _ = w.Sync() + fmt.Fprintf(cmd.OutOrStdout(), "interrupted after %d messages, progress saved to %s\n", total, out) + return err + } + return err + } + if err := w.Sync(); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "exported %d messages to %s\n", total, out) + return nil + }, + } + cmd.Flags().StringVar(&out, "out", "", "output store file with .art extension (required)") + cmd.Flags().DurationVar(&drainTimeout, "drain-timeout", 5*time.Second, "idle time before a queue is considered empty") + cmd.Flags().IntVar(&batch, "batch", 100, "persist/ack batch size") + _ = cmd.MarkFlagRequired("out") + return cmd +} diff --git a/internal/cli/export_integration_test.go b/internal/cli/export_integration_test.go new file mode 100644 index 0000000..313a977 --- /dev/null +++ b/internal/cli/export_integration_test.go @@ -0,0 +1,32 @@ +// internal/cli/export_integration_test.go +package cli + +import ( + "os" + "path/filepath" + "testing" +) + +func TestExportCommandCreatesStore(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + // Reuse broker package's container helper via a fresh broker for seeding. + props := startArtemisForCLI(t) + seedQueue(t, props, "orders", []string{"a", "b"}) + + out := filepath.Join(t.TempDir(), "dump.artx") + root := NewRootCmd() + root.SetArgs([]string{ + "export", "--out", out, + "--url", props.URL, "-u", props.Username, "-p", props.Password, + "--drain-timeout", "3s", + }) + if err := root.Execute(); err != nil { + t.Fatalf("export: %v", err) + } + fi, err := os.Stat(out) + if err != nil || fi.Size() <= 5 { + t.Fatalf("store not written: %v size=%v", err, fi) + } +} diff --git a/internal/cli/health.go b/internal/cli/health.go new file mode 100644 index 0000000..0476dc9 --- /dev/null +++ b/internal/cli/health.go @@ -0,0 +1,42 @@ +package cli + +import ( + "context" + "fmt" + + "github.com/martikan/artemisctl/internal/broker" + "github.com/spf13/cobra" +) + +func newHealthCmd() *cobra.Command { + return &cobra.Command{ + Use: "health", + Short: "Report resource usage and a redelivery-readiness verdict", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx, cancel := connectCtx(cmd, context.Background()) + defer cancel() + c, err := broker.Connect(ctx, connProps(cmd)) + if err != nil { + return err + } + defer c.Close(ctx) + h, err := c.CheckHealth(ctx) + if err != nil { + return err + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "disk: %.1f%%\n", h.DiskUsagePct) + fmt.Fprintf(out, "memory: %.1f%%\n", h.MemoryUsagePct) + fmt.Fprintf(out, "blocking: %v\n", h.Blocking) + fmt.Fprintf(out, "verdict: %s\n", h.Verdict) + switch h.Verdict { + case broker.Critical: + cmd.SilenceErrors = true + return fmt.Errorf("broker CRITICAL") + case broker.Degraded: + // exit 0 but visible; keep simple + } + return nil + }, + } +} diff --git a/internal/cli/produce.go b/internal/cli/produce.go new file mode 100644 index 0000000..44e6a6f --- /dev/null +++ b/internal/cli/produce.go @@ -0,0 +1,101 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/Azure/go-amqp" + "github.com/martikan/artemisctl/internal/broker" + "github.com/spf13/cobra" +) + +func newProduceCmd() *cobra.Command { + var queue, file string + var count, size, rate, workers int + var properties []string + + cmd := &cobra.Command{ + Use: "produce", + Short: "Send messages to a queue (generated test data or from a JSON file)", + RunE: func(cmd *cobra.Command, _ []string) error { + if workers < 1 { + return fmt.Errorf("--workers must be >= 1, got %d", workers) + } + props, err := parseProperties(properties) + if err != nil { + return err + } + + var msgs []*amqp.Message + if file != "" { + data, err := os.ReadFile(file) + if err != nil { + return fmt.Errorf("read message file: %w", err) + } + msgs, err = broker.ParseArtemisMessages(data, props) + if err != nil { + return err + } + } else { + msgs = broker.GenerateMessages(count, size, props) + } + if len(msgs) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "no messages to send") + return nil + } + + // Bound Connect by --timeout; the send loop runs under a separate + // SIGINT-cancelable context so Ctrl-C stops cleanly between messages. + connCtx, connCancel := connectCtx(cmd, context.Background()) + defer connCancel() + c, err := broker.Connect(connCtx, connProps(cmd)) + if err != nil { + return err + } + defer c.Close(context.Background()) + + ctx, stop := signalCtx() + defer stop() + + sent, err := c.Produce(ctx, queue, msgs, rate, workers, nil) + if err != nil { + if errors.Is(err, context.Canceled) { + fmt.Fprintf(cmd.OutOrStdout(), "interrupted after %d of %d messages\n", sent, len(msgs)) + return err + } + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "produced %d messages to %s\n", sent, queue) + return nil + }, + } + cmd.Flags().StringVar(&queue, "queue", "", "target queue (required)") + cmd.Flags().StringVar(&file, "file", "", "JSON file of Artemis-console-shaped messages to send (array)") + cmd.Flags().IntVar(&count, "count", 1, "number of generated messages (ignored with --file)") + cmd.Flags().IntVar(&size, "size", 256, "generated message body size in bytes (ignored with --file)") + cmd.Flags().IntVar(&rate, "rate", 0, "max messages per second in total (0 = unlimited)") + cmd.Flags().IntVar(&workers, "workers", 1, "parallel sender sessions (1 = ordered, sequential)") + cmd.Flags().StringArrayVar(&properties, "property", nil, "application property k=v set on every message (repeatable)") + _ = cmd.MarkFlagRequired("queue") + return cmd +} + +// parseProperties turns repeated "k=v" flag values into an application-property +// map. Values are always strings; the key is everything before the first "=". +func parseProperties(pairs []string) (map[string]interface{}, error) { + if len(pairs) == 0 { + return nil, nil + } + props := make(map[string]interface{}, len(pairs)) + for _, p := range pairs { + k, v, ok := strings.Cut(p, "=") + if !ok || k == "" { + return nil, fmt.Errorf("invalid --property %q, expected k=v", p) + } + props[k] = v + } + return props, nil +} diff --git a/internal/cli/produce_test.go b/internal/cli/produce_test.go new file mode 100644 index 0000000..9ceae3a --- /dev/null +++ b/internal/cli/produce_test.go @@ -0,0 +1,20 @@ +// internal/cli/produce_test.go +package cli + +import ( + "bytes" + "strings" + "testing" +) + +func TestProduceRejectsInvalidWorkers(t *testing.T) { + root := NewRootCmd() + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs([]string{"produce", "--queue", "q", "--workers", "0"}) + err := root.Execute() + if err == nil || !strings.Contains(err.Error(), "--workers") { + t.Fatalf("want --workers validation error, got %v", err) + } +} diff --git a/internal/cli/redeliver.go b/internal/cli/redeliver.go new file mode 100644 index 0000000..7f9874c --- /dev/null +++ b/internal/cli/redeliver.go @@ -0,0 +1,72 @@ +package cli + +import ( + "context" + "errors" + "fmt" + + "github.com/martikan/artemisctl/internal/broker" + "github.com/spf13/cobra" +) + +func newRedeliverCmd() *cobra.Command { + var in, queueOverride string + var force bool + + cmd := &cobra.Command{ + Use: "redeliver", + Short: "Replay a store file back to the broker (health-gated, resumable, dedup)", + RunE: func(cmd *cobra.Command, _ []string) error { + // Bound Connect + the pre-flight health check by --timeout so an + // unresponsive broker fails fast. + connCtx, connCancel := connectCtx(cmd, context.Background()) + defer connCancel() + c, err := broker.Connect(connCtx, connProps(cmd)) + if err != nil { + return err + } + defer c.Close(context.Background()) + + h, err := c.CheckHealth(connCtx) + if err != nil { + return err + } + if h.Verdict == broker.Critical && !force { + return fmt.Errorf("broker health CRITICAL (disk %.1f%%, mem %.1f%%); refuse to redeliver (use --force)", + h.DiskUsagePct, h.MemoryUsagePct) + } + + // The replay loop runs under a separate long-lived context that is + // NOT capped by --timeout (a large replay may exceed it) but IS + // cancelable via SIGINT. Per-record checkpointing means a clean + // cancellation leaves an exact resume point. + ctx, stop := signalCtx() + defer stop() + + n, coreSkipped, err := c.Redeliver(ctx, in, broker.RedeliverOpts{QueueOverride: queueOverride}, + func(sent int) { + if sent%100 == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "redelivered %d...\n", sent) + } + }) + if err != nil { + if errors.Is(err, context.Canceled) { + fmt.Fprintf(cmd.OutOrStdout(), "interrupted after %d messages, progress saved (resume with the same command)\n", n) + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "redelivered %d messages before stopping: %v\n", n, err) + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "redelivered %d messages from %s\n", n, in) + if coreSkipped > 0 { + fmt.Fprintf(cmd.OutOrStdout(), "warning: %d Core-protocol record(s) could not be converted to AMQP and were skipped (left in the store)\n", coreSkipped) + } + return nil + }, + } + cmd.Flags().StringVar(&in, "in", "", "input store file (required)") + cmd.Flags().StringVar(&queueOverride, "queue", "", "redirect all messages to this queue") + cmd.Flags().BoolVar(&force, "force", false, "redeliver even if broker health is CRITICAL") + _ = cmd.MarkFlagRequired("in") + return cmd +} diff --git a/internal/cli/root.go b/internal/cli/root.go new file mode 100644 index 0000000..69c1b71 --- /dev/null +++ b/internal/cli/root.go @@ -0,0 +1,84 @@ +// Package cli wires the artemisctl command tree. It builds the cobra commands +// (status, health, browse, produce, export, redeliver), reads the shared +// connection flags, and translates each command into calls on the broker +// package. The helpers here — connProps, connectCtx, signalCtx — centralize +// how every command connects and how it scopes cancellation: a --timeout only +// bounds the initial connect, while long-running drains and replays run under +// a SIGINT-cancelable context instead. +package cli + +import ( + "context" + "os" + "os/signal" + "time" + + "github.com/martikan/artemisctl/internal/broker" + "github.com/spf13/cobra" +) + +// NewRootCmd builds the root artemisctl command with its persistent connection +// flags and every subcommand attached, ready to Execute. +func NewRootCmd() *cobra.Command { + root := &cobra.Command{ + Use: "artemisctl", + Short: "Manage and recover Apache ActiveMQ Artemis brokers", + SilenceUsage: true, + SilenceErrors: true, + } + pf := root.PersistentFlags() + pf.String("url", "127.0.0.1:61616", "AMQP 1.0 broker address host:port") + pf.StringP("username", "u", "artemis", "broker username") + pf.StringP("password", "p", "artemis", "broker password (prefer ARTEMIS_PASSWORD env)") + pf.Duration("timeout", 30*time.Second, "max time to establish the broker connection before failing fast") + root.AddCommand(newStatusCmd()) + root.AddCommand(newExportCmd()) + root.AddCommand(newHealthCmd()) + root.AddCommand(newRedeliverCmd()) + root.AddCommand(newBrowseCmd()) + root.AddCommand(newProduceCmd()) + root.AddCommand(newCordonCmd()) + root.AddCommand(newUncordonCmd()) + root.AddCommand(newSalvageCmd()) + return root +} + +// resolvePassword returns ARTEMIS_PASSWORD when set, else the flag value. +func resolvePassword(flagVal string) string { + if env := os.Getenv("ARTEMIS_PASSWORD"); env != "" { + return env + } + return flagVal +} + +// connProps reads the persistent flags + ARTEMIS_PASSWORD into ConnectionProps. +func connProps(cmd *cobra.Command) broker.ConnectionProps { + url, _ := cmd.Flags().GetString("url") + user, _ := cmd.Flags().GetString("username") + pass, _ := cmd.Flags().GetString("password") + return broker.ConnectionProps{URL: url, Username: user, Password: resolvePassword(pass)} +} + +// connectCtx derives a context bounded by the --timeout flag. It always bounds +// broker.Connect so an unresponsive broker fails fast instead of hanging +// forever; for the short read-only commands (status/health/browse) the same +// context bounds the whole operation, since those complete quickly. The caller +// must defer the returned cancel. It intentionally does NOT bound long-running +// work (export drain / redeliver replay), which may legitimately exceed the +// timeout — those pass a separate unbounded/signal context to the drain/replay +// loop. +func connectCtx(cmd *cobra.Command, parent context.Context) (context.Context, context.CancelFunc) { + d, _ := cmd.Flags().GetDuration("timeout") + if d <= 0 { + return context.WithCancel(parent) + } + return context.WithTimeout(parent, d) +} + +// signalCtx returns a context that is canceled on SIGINT, for the long-running +// drain/replay loops so Ctrl-C returns gracefully with a durable checkpoint +// rather than aborting an in-flight send. The caller must defer the returned +// stop. +func signalCtx() (context.Context, context.CancelFunc) { + return signal.NotifyContext(context.Background(), os.Interrupt) +} diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go new file mode 100644 index 0000000..5050ccf --- /dev/null +++ b/internal/cli/root_test.go @@ -0,0 +1,46 @@ +package cli + +import ( + "bytes" + "testing" + "time" +) + +func TestRootCmdHasConnectionFlags(t *testing.T) { + cmd := NewRootCmd() + for _, name := range []string{"url", "username", "password"} { + if cmd.PersistentFlags().Lookup(name) == nil { + t.Fatalf("missing persistent flag %q", name) + } + } +} + +// TestRootCmdHasTimeoutFlag is the I2 regression: a --timeout persistent flag +// must exist and default to 30s so an unresponsive broker fails the Connect +// fast instead of hanging on context.Background forever. +func TestRootCmdHasTimeoutFlag(t *testing.T) { + cmd := NewRootCmd() + f := cmd.PersistentFlags().Lookup("timeout") + if f == nil { + t.Fatal("missing persistent flag \"timeout\"") + } + if f.DefValue != "30s" { + t.Fatalf("timeout default = %q, want 30s", f.DefValue) + } + d, err := cmd.PersistentFlags().GetDuration("timeout") + if err != nil { + t.Fatalf("GetDuration: %v", err) + } + if d != 30*time.Second { + t.Fatalf("timeout parsed = %v, want 30s", d) + } +} + +func TestRootCmdHelpRuns(t *testing.T) { + cmd := NewRootCmd() + cmd.SetArgs([]string{"--help"}) + cmd.SetOut(&bytes.Buffer{}) + if err := cmd.Execute(); err != nil { + t.Fatalf("help failed: %v", err) + } +} diff --git a/internal/cli/salvage.go b/internal/cli/salvage.go new file mode 100644 index 0000000..9824985 --- /dev/null +++ b/internal/cli/salvage.go @@ -0,0 +1,339 @@ +package cli + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "syscall" + + "github.com/martikan/artemisctl/internal/journal" + "github.com/martikan/artemisctl/internal/store" + "github.com/spf13/cobra" +) + +// newSalvageCmd builds `artemisctl salvage`: the offline counterpart to +// `export` that reads a *stopped* broker's data directory straight off disk +// (bindings + message journal + large-messages + paging, via +// internal/journal's Salvage pipeline) and writes every recoverable message +// to a local .artx store. Unlike every other command in this tree it never +// dials a broker, so it deliberately ignores the persistent --url/--username/ +// --password/--timeout connection flags -- that is called out explicitly in +// the Long help below so `--help` output doesn't mislead an operator into +// thinking those flags matter here. +func newSalvageCmd() *cobra.Command { + var dataDir, bindingsDir, journalDir, largeDir, pagingDir, out string + var force, allowSkips bool + + cmd := &cobra.Command{ + Use: "salvage", + Short: "Recover messages from a stopped broker's data directory into a local store (offline)", + Long: "salvage reads a broker data directory directly off disk -- the bindings and message " + + "journals, large-message bodies, and paging state -- and writes every recoverable " + + "message to a local .artx store for a later `redeliver`, without needing a running " + + "broker. It IGNORES the persistent --url/--username/--password/--timeout connection " + + "flags entirely: this command never dials the broker. It refuses to run against a data " + + "directory whose broker process is still alive (server.lock held); pass --force to " + + "override that guard, but export from a live broker is the safer path when one is " + + "available. Core-protocol messages are decoded and exported alongside AMQP messages " + + "(redeliver converts them back to AMQP on send). The summary printed at the end " + + "itemizes both skipped records (unsupported data) and diagnostics (corruption incidents and other " + + "notes); by default the command exits non-zero whenever either skips or corruption " + + "diagnostics are present, so a script can't miss them -- pass --allow-skips to accept " + + "them and exit 0 anyway.", + RunE: func(cmd *cobra.Command, _ []string) error { + if dataDir == "" && (bindingsDir == "" || journalDir == "") { + return errors.New("salvage: --data is required unless both --bindings and --journal are set") + } + bindings := resolveSalvageSubDir(bindingsDir, dataDir, "bindings") + journalD := resolveSalvageSubDir(journalDir, dataDir, "journal") + largeMessages := resolveSalvageSubDir(largeDir, dataDir, "large-messages") + paging := resolveSalvageSubDir(pagingDir, dataDir, "paging") + + if err := requireExistingDir(bindings); err != nil { + return fmt.Errorf("salvage: bindings dir: %w", err) + } + if err := requireExistingDir(journalD); err != nil { + return fmt.Errorf("salvage: journal dir: %w", err) + } + + // Pre-check the FINAL --out path before doing any work, mirroring + // store.NewWriter's own refusal semantics: re-running salvage into + // an already-populated store must never silently clobber it. This + // check has to happen here (not just rely on NewWriter on the + // .partial path below) because the .partial file is a different + // path than --out, so NewWriter alone would never see --out at all. + if fi, err := os.Stat(out); err == nil && fi.Size() > 0 { + return fmt.Errorf("store %s already exists and is non-empty; choose a new path to avoid overwriting drained data", out) + } else if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("salvage: stat %s: %w", out, err) + } + + if !force { + if err := checkLiveBroker(journalD, dataDir); err != nil { + return err + } + } + + partial := out + ".partial" + // A killed/crashed prior run can leave a non-empty .partial + // behind; store.NewWriter refuses to reuse a non-empty file (its + // own guard exists to protect a REAL store, e.g. --out itself, + // against accidental truncation -- see the --out check above). + // .partial is never a source of truth here -- it only ever holds + // bytes from a run that did not reach the final rename to --out + // -- so it is always safe to discard before starting this run's + // write. Ignore a not-exist error; anything else surfaces below + // when NewWriter itself fails to open the path. + if err := os.Remove(partial); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("salvage: remove stale partial store %s: %w", partial, err) + } + w, err := store.NewWriter(partial) + if err != nil { + return fmt.Errorf("salvage: open partial store %s: %w", partial, err) + } + + // maxAMQPLen tracks, per destination queue, the largest single + // record's encoded AMQP byte length seen during this run. Summary + // (Task 10's frozen interface) only carries a single global + // LargestBytes -- it has no per-queue breakdown, and store.Record + // itself carries no "this came from a large message" flag either, + // so there is no way to ask the journal package which queue the + // large message landed on. Tracking the max length here, at the + // CLI layer, recovers that attribution for realistic data without + // touching Task 10's interfaces: a large message's encoded length + // dwarfs every ordinary message's, so the queue with the biggest + // single record is -- in practice -- the queue that received the + // large message. See the print-format decision note in the task + // report for the full reasoning. + maxAMQPLen := make(map[string]int) + summary, salvageErr := journal.Salvage(journal.Options{ + Bindings: bindings, + Journal: journalD, + LargeMessages: largeMessages, + Paging: paging, + }, func(r store.Record) error { + n := len(r.AMQP) + if r.Kind == store.KindCore { + n = len(r.CorePayload) + } + if n > maxAMQPLen[r.Queue] { + maxAMQPLen[r.Queue] = n + } + return w.Append(r) + }) + if salvageErr != nil { + _ = w.Close() + _ = os.Remove(partial) + return salvageErr + } + + total := summary.Total() + if total == 0 { + // spec: never leave an empty store behind. + _ = w.Close() + _ = os.Remove(partial) + printSalvageSummary(cmd.OutOrStdout(), summary, out, 0, "") + // The skips/corruption gate applies here too (finding I1): a + // journal that salvages nothing but still has skips or + // corruption diagnostics (e.g. a data dir holding only + // Core-protocol messages, or large messages whose bodies are + // all missing) is the worst case for a scripted recovery -- + // exiting 0 would make that silently indistinguishable from + // "genuinely nothing here to salvage". + if (summary.HasSkips() || summary.HasCorruption()) && !allowSkips { + return errors.New("skips or corruption present — failing (pass --allow-skips to override)") + } + return nil + } + + if err := w.Sync(); err != nil { + _ = w.Close() + return fmt.Errorf("salvage: sync partial store: %w", err) + } + if err := w.Close(); err != nil { + return fmt.Errorf("salvage: close partial store: %w", err) + } + if err := os.Rename(partial, out); err != nil { + return fmt.Errorf("salvage: rename %s to %s: %w", partial, out, err) + } + + largestQueue := "" + if summary.Large > 0 { + names := make([]string, 0, len(maxAMQPLen)) + for q := range maxAMQPLen { + names = append(names, q) + } + sort.Strings(names) + best := -1 + for _, q := range names { + if maxAMQPLen[q] > best { + best = maxAMQPLen[q] + largestQueue = q + } + } + } + printSalvageSummary(cmd.OutOrStdout(), summary, out, total, largestQueue) + + if (summary.HasSkips() || summary.HasCorruption()) && !allowSkips { + return errors.New("skips or corruption present — failing (pass --allow-skips to override)") + } + return nil + }, + } + + cmd.Flags().StringVar(&dataDir, "data", "", "broker data directory (parent of bindings/journal/large-messages/paging); optional if --bindings and --journal are both given") + cmd.Flags().StringVar(&bindingsDir, "bindings", "", "bindings journal dir (default /bindings)") + cmd.Flags().StringVar(&journalDir, "journal", "", "message journal dir (default /journal)") + cmd.Flags().StringVar(&largeDir, "large-messages", "", "large-messages dir (default /large-messages)") + cmd.Flags().StringVar(&pagingDir, "paging", "", "paging dir (default /paging)") + cmd.Flags().StringVar(&out, "out", "", "output store file (required)") + cmd.Flags().BoolVar(&force, "force", false, "skip the live-broker (server.lock) guard") + cmd.Flags().BoolVar(&allowSkips, "allow-skips", false, "exit 0 even though some messages were skipped or corruption diagnostics were reported (unsupported/corrupt data)") + _ = cmd.MarkFlagRequired("out") + + return cmd +} + +// resolveSalvageSubDir returns explicit if set, else dataDir/sub. +func resolveSalvageSubDir(explicit, dataDir, sub string) string { + if explicit != "" { + return explicit + } + return filepath.Join(dataDir, sub) +} + +// requireExistingDir returns an error if dir does not exist, is unreadable, +// or is not a directory -- the CLI's own fast-fail check ahead of +// journal.Salvage's identical (but differently worded) internal check, so a +// missing --bindings/--journal dir is reported before any other validation +// or work (live-broker probe, partial-store creation) happens. +func requireExistingDir(dir string) error { + fi, err := os.Stat(dir) + if err != nil { + return err + } + if !fi.IsDir() { + return fmt.Errorf("%s: not a directory", dir) + } + return nil +} + +// checkLiveBroker guards against running salvage against a data directory +// whose broker process is still alive. Artemis's JournalStorageManager holds +// an exclusive flock on server.lock for as long as the broker is up. +// +// journalDir is the message-journal sub-dir (e.g. /journal); dataDir +// is the broker data directory that is its parent (e.g. /data) -- +// dataDir may be "" when the caller supplied --bindings/--journal directly +// without --data. The instance root is dataDir's own parent (e.g. +// ), since the standard broker layout is +// /data/{bindings,journal,...} alongside /server.lock. +// +// Three candidate locations are probed, in order, and the first one that +// exists on disk is used: +// 1. /server.lock -- fixture-verified real location for the +// artemis-2.42-data fixture this package tests against. +// 2. /server.lock (only if dataDir != "") -- kept for backward +// compatibility; harmless to probe even though no observed layout uses +// it, since a probe-only check on a path that never exists is a no-op. +// 3. filepath.Dir(dataDir)/server.lock, i.e. the instance root (only if +// dataDir != "") -- the layout the task brief calls out explicitly. +// +// If none of the candidates exist, there is nothing to check against and +// salvage proceeds -- this offline tool never creates the lock file itself, +// only probes an existing one. +func checkLiveBroker(journalDir, dataDir string) error { + candidates := []string{filepath.Join(journalDir, "server.lock")} + if dataDir != "" { + candidates = append(candidates, + filepath.Join(dataDir, "server.lock"), + filepath.Join(filepath.Dir(dataDir), "server.lock"), + ) + } + var lockPath string + for _, c := range candidates { + if fi, err := os.Stat(c); err == nil && !fi.IsDir() { + lockPath = c + break + } + } + if lockPath == "" { + return nil + } + + f, err := os.OpenFile(lockPath, os.O_RDONLY, 0) + if err != nil { + return fmt.Errorf("salvage: open %s for live-broker check: %w", lockPath, err) + } + defer f.Close() + + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if errors.Is(err, syscall.EWOULDBLOCK) { + return errors.New("broker appears to be running (server.lock held) — use `export` for a live broker, or --force") + } + return fmt.Errorf("salvage: flock %s: %w", lockPath, err) + } + // The probe only ever wants to know whether the lock is free; release it + // immediately rather than holding it for the run. + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + return nil +} + +// printSalvageSummary writes the operator-facing report: the headline count +// (and destination, unless nothing was recovered), one aligned line per +// destination queue, and -- if any messages were unrecoverable -- a +// "skipped:" section listing why, followed by -- if anything else worth +// flagging happened -- a "diagnostics:" section (corruption incidents and +// benign notes alike; see journal.Summary.Diags). largestQueue, when +// non-empty, names the queue whose line gets the "(largest N)" annotation +// (see the caller's maxAMQPLen comment for how that queue is chosen). +func printSalvageSummary(w io.Writer, summary journal.Summary, out string, total int, largestQueue string) { + if total == 0 { + fmt.Fprintln(w, "salvaged 0 messages (nothing recoverable; no store written)") + } else { + fmt.Fprintf(w, "salvaged %d messages to %s\n", total, out) + if summary.Core > 0 { + fmt.Fprintf(w, " (%d Core-protocol messages decoded; redeliver converts them to AMQP)\n", summary.Core) + } + } + + names := make([]string, 0, len(summary.PerQueue)) + maxNameLen := 0 + for name := range summary.PerQueue { + names = append(names, name) + if len(name) > maxNameLen { + maxNameLen = len(name) + } + } + sort.Strings(names) + width := maxNameLen + 2 + for _, name := range names { + line := fmt.Sprintf(" %-*s%d", width, name, summary.PerQueue[name]) + if name == largestQueue { + line += fmt.Sprintf(" (largest %s)", formatKiB(summary.LargestBytes)) + } + fmt.Fprintln(w, line) + } + + if summary.HasSkips() { + fmt.Fprintln(w, "skipped:") + for _, s := range summary.Skips { + fmt.Fprintf(w, " %s\n", s) + } + } + + if len(summary.Diags) > 0 { + fmt.Fprintln(w, "diagnostics:") + for _, d := range summary.Diags { + fmt.Fprintf(w, " %s\n", d) + } + } +} + +// formatKiB renders a byte count as "N.N KiB", one decimal place. +func formatKiB(n int64) string { + return fmt.Sprintf("%.1f KiB", float64(n)/1024) +} diff --git a/internal/cli/salvage_integration_test.go b/internal/cli/salvage_integration_test.go new file mode 100644 index 0000000..aa8b6ce --- /dev/null +++ b/internal/cli/salvage_integration_test.go @@ -0,0 +1,371 @@ +// internal/cli/salvage_integration_test.go +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Azure/go-amqp" + "github.com/martikan/artemisctl/internal/broker" + "github.com/martikan/artemisctl/internal/store" +) + +// manifestEntry / manifestFile mirror internal/journal/testdata/manifest.json's +// schema. internal/journal/message_test.go carries the canonical copy of this +// same shape (manifestEntry/manifestFile); it is unexported to that package +// (package journal, not journal_test), so this is a small local duplicate per +// the task brief. +type manifestEntry struct { + BodySha256 string `json:"bodySha256"` + BodyLen int `json:"bodyLen"` + Props map[string]any `json:"props,omitempty"` + ScheduledAtMs int64 `json:"scheduledAtMs,omitempty"` +} + +type manifestFile struct { + Queues map[string][]manifestEntry `json:"queues"` +} + +func loadSalvageManifest(t *testing.T) manifestFile { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "journal", "testdata", "manifest.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var m manifestFile + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("unmarshal manifest: %v", err) + } + return m +} + +// captureSink is a broker.RecordSink that keeps every drained record in +// memory in delivery order, for DrainQueue-based verification. +type captureSink struct { + records []store.Record +} + +func (s *captureSink) Append(r store.Record) error { + s.records = append(s.records, r) + return nil +} +func (s *captureSink) Sync() error { return nil } + +// destroyQueue removes a queue entirely -- both any remaining messages and +// the queue definition itself -- via a raw broker.destroyQueue management +// call, replicating internal/broker's unexported callManagement locally +// (package cli cannot reach it). This exists purely for final cleanup: this +// test's redirect queue always ends up holding one message DrainQueue can +// never remove (the fixture's scheduled record, see the CAUTION below, is +// not deliverable until 2100-01-01), so without an explicit destroy the +// queue -- and that stuck message -- would live forever on the shared, +// never-terminated broker container (see brokertest.Shared's docstring: +// "state accumulates"). That is not hypothetical: earlier iterations of this +// test left exactly such orphaned salvage.e2e.* queues behind, and their +// accumulation broke internal/broker's TestDrainAllEnumeratesAndDrainsEveryQueue +// by exceeding its bounded drain-everything deadline. Confirmed working +// against the shared broker before wiring it in here. +func destroyQueue(ctx context.Context, sess *amqp.Session, name string) error { + recv, err := sess.NewReceiver(ctx, "", &amqp.ReceiverOptions{DynamicAddress: true}) + if err != nil { + return fmt.Errorf("create reply receiver: %w", err) + } + defer recv.Close(context.Background()) + replyTo := recv.Address() + + sender, err := sess.NewSender(ctx, "activemq.management", nil) + if err != nil { + return fmt.Errorf("create management sender: %w", err) + } + defer sender.Close(context.Background()) + + msg := &amqp.Message{ + Value: fmt.Sprintf(`[%q, true, true]`, name), // [queueName, removeConsumers, autoDeleteAddress] + Properties: &amqp.MessageProperties{ReplyTo: &replyTo}, + ApplicationProperties: map[string]interface{}{ + "_AMQ_ResourceName": "broker", + "_AMQ_OperationName": "destroyQueue", + }, + } + if err := sender.Send(ctx, msg, nil); err != nil { + return fmt.Errorf("send destroyQueue request: %w", err) + } + reply, err := recv.Receive(ctx, nil) + if err != nil { + return fmt.Errorf("receive destroyQueue reply: %w", err) + } + _ = recv.AcceptMessage(ctx, reply) + if ok, present := reply.ApplicationProperties["_AMQ_OperationSucceeded"].(bool); present && !ok { + return fmt.Errorf("broker rejected destroyQueue(%s): %v", name, reply.Value) + } + return nil +} + +// TestSalvageE2E proves the full offline-recovery workflow against a real +// broker: salvage a broker data directory into a store file, redeliver it +// into a single unique redirect queue, and verify every recoverable record +// made it across with its body (and, for the props sample, its application +// properties) intact. +func TestSalvageE2E(t *testing.T) { + if testing.Short() { + t.Skip("skip integration in -short") + } + + dir := salvageFixtureDir(t) + storePath := filepath.Join(t.TempDir(), "rescue.artx") + + // --force is deliberately NOT used here: the fixture's + // data/journal/server.lock exists but is unheld (nothing holds the + // flock), so checkLiveBroker's guard already passes on its own -- see + // TestSalvageFixtureSuccess, which salvages this same fixture without + // --force. Exercising the no-force path matches the real recovery + // workflow: --force is only needed when the live-broker guard actually + // fires, which it does not for a harvested/copied data directory. + stdout, err := runSalvage(t, "--data", dir, "--out", storePath) + if err != nil { + t.Fatalf("salvage: %v\noutput:\n%s", err, stdout) + } + const wantTotal = 512 // 5 plain + 5 props + 1 scheduled + 1 large + 500 paged + wantHeadline := fmt.Sprintf("salvaged %d messages to %s", wantTotal, storePath) + if !strings.Contains(stdout, wantHeadline) { + t.Fatalf("salvage stdout missing %q; got:\n%s", wantHeadline, stdout) + } + + man := loadSalvageManifest(t) + // CAUTION (dedup): Redeliver dedups on the broker via _AMQ_DUPL_ID, a + // content hash salted with the record's OWN originating queue at salvage + // time (not the redirect queue -- see store.DedupID and internal/journal's + // salvage path), so collapsing every record onto one redirect queue only + // risks losing a message if two records shared a hash to begin with. + // Confirm the premise holds for this fixture: every body sha256 in the + // manifest is unique, so there is no real duplicate content for the + // broker's duplicate-detection to legitimately collapse. + bodyOrigin := map[string]string{} // body sha256 -> manifest queue name + for q, entries := range man.Queues { + for _, e := range entries { + if prev, ok := bodyOrigin[e.BodySha256]; ok { + t.Fatalf("manifest has duplicate body sha256 %s on both %s and %s; dedup-safety assumption violated", e.BodySha256, prev, q) + } + bodyOrigin[e.BodySha256] = q + } + } + + props := startArtemisForCLI(t) + redirectQueue := fmt.Sprintf("salvage.e2e.%d", time.Now().UnixNano()) + + runConn := func(t *testing.T, args ...string) (string, error) { + t.Helper() + conn := []string{"--url", props.URL, "-u", props.Username, "-p", props.Password} + return runCmdStdin(t, "", append(args, conn...)...) + } + + redelivOut, err := runConn(t, "redeliver", "--in", storePath, "--queue", redirectQueue) + if err != nil { + t.Fatalf("redeliver: %v\noutput:\n%s", err, redelivOut) + } + wantRedelivHeadline := fmt.Sprintf("redelivered %d messages from %s", wantTotal, storePath) + if !strings.Contains(redelivOut, wantRedelivHeadline) { + t.Fatalf("redeliver stdout missing %q; got:\n%s", wantRedelivHeadline, redelivOut) + } + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + c, err := broker.Connect(ctx, props) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer c.Close(context.Background()) + + // Guaranteed cleanup: destroy the redirect queue at test end, whatever + // the outcome. Registered right after Connect so it runs even if any + // assertion below Fatals, and (defers being LIFO) BEFORE the deferred + // c.Close, so the client's session is still open when it fires. + // Best-effort with t.Logf, not t.Errorf: a cleanup hiccup must not turn + // a passing verification of the recovery pipeline into a failure -- but + // it is loud in the log so an operator knows the shared broker needs a + // manual sweep of this one queue. + defer func() { + cctx, ccancel := context.WithTimeout(context.Background(), 15*time.Second) + defer ccancel() + if err := destroyQueue(cctx, c.Session(), redirectQueue); err != nil { + t.Logf("cleanup: destroyQueue(%s) failed (shared broker may need a manual sweep): %v", redirectQueue, err) + } + }() + + // The broker-reported queue depth is a true aggregate stat (Artemis's + // listQueues management op), unaffected by browse-page-size limits or + // consumer credit -- it is the deterministic proof that every one of the + // 512 salvaged records actually reached the broker under the redirect + // queue, matching the salvage summary total exactly. + qstats, err := c.ListQueues(ctx) + if err != nil { + t.Fatalf("ListQueues: %v", err) + } + var brokerCount int64 = -1 + for _, q := range qstats { + if q.Name == redirectQueue { + brokerCount = q.MessageCount + } + } + if brokerCount != wantTotal { + t.Fatalf("broker-reported message count for %s = %d, want %d", redirectQueue, brokerCount, wantTotal) + } + + // CAUTION (scheduled message) + why verification uses DrainQueue rather + // than BrowseQueue: + // + // The fixture's salvage.scheduled record carries an x-opt-delivery-time + // annotation of 4102444800000ms, i.e. 2100-01-01T00:00:00Z (see + // manifest.json) -- far in the future relative to any real test run. + // Redeliver preserves the record byte-for-byte (it round-trips the raw + // AMQP message and only adds the _AMQ_DUPL_ID property), so Artemis + // honors the annotation and holds that one message as scheduled: it is + // enqueued (counted in brokerCount above) but not delivery-eligible to + // any consumer, ever, until then. So exactly wantTotal-1 = 511 records + // are actually retrievable right now. + // + // A first attempt used BrowseQueue (a non-destructive receive-then- + // release peek, see internal/broker/browse.go) to both count and sample + // those 511. It does not work cleanly at this queue depth against the + // shared broker's settings: brokertest.PermissiveWildcardSettings sets + // managementBrowsePageSize=200, so BrowseQueue's internal queueCount() + // (via listMessagesAsJSON) caps out at 200 regardless of the requested + // limit -- and that same settings blob sets redeliveryDelay=0 (needed + // elsewhere so back-to-back browses in other tests don't hit the + // "recently-released invisibility window" also documented in + // browse.go). With no redelivery delay, a message released mid-peek can + // be redispatched to the SAME manual-credit receiver before every + // distinct message has been seen once, producing real, observed + // duplicate deliveries within a single peek (confirmed empirically: + // requesting 200 yielded only ~189-190 distinct message-ids across + // repeated runs). That makes an exact count/bucket assertion via + // BrowseQueue flaky at this scale -- not a bug in this test's logic, a + // real interaction between two address-settings this shared broker needs + // for other tests. + // + // DrainQueue sidesteps both problems: it has no page-size cap (it is a + // plain AMQP receive loop, not listMessagesAsJSON-backed) and it ACCEPTs + // each message instead of releasing it, so a message is durably removed + // on first delivery -- no redelivery, no duplicates, exactly-once by + // construction. It is also the same call the per-test reset contract + // uses to leave the shared broker clean, so this one exhaustive drain + // serves as BOTH the deterministic, full (not sampled) content + // verification AND the message-level cleanup; the still-scheduled + // record it cannot touch is removed by the deferred destroyQueue. + const wantDrainable = wantTotal - 1 + + sink := &captureSink{} + dctx, dcancel := context.WithTimeout(context.Background(), 60*time.Second) + n, err := c.DrainQueue(dctx, redirectQueue, sink, 1*time.Second, 200) + dcancel() + if err != nil { + t.Fatalf("DrainQueue: %v", err) + } + if n != wantDrainable { + t.Fatalf("DrainQueue drained %d messages, want %d (%d total minus the not-yet-due scheduled record, which cannot be drained until 2100-01-01)", + n, wantDrainable, wantTotal) + } + if len(sink.records) != wantDrainable { + t.Fatalf("captured %d records, want %d", len(sink.records), wantDrainable) + } + + // After the drain exactly one message must remain: the scheduled record, + // which no consumer can remove until its far-future delivery time. (The + // deferred destroyQueue above takes the whole queue -- stuck record + // included -- with it at test end.) + // + // Artemis's listQueues message-count is settled asynchronously after a + // batch of acks (empirically observed: immediately after DrainQueue + // returns it can still reflect a partially-decremented count, converging + // to the true value within ~1s), so poll briefly instead of asserting on + // the first read. + var afterCount int64 = -1 + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { + afterQstats, err := c.ListQueues(ctx) + if err != nil { + t.Fatalf("ListQueues after drain: %v", err) + } + for _, q := range afterQstats { + if q.Name == redirectQueue { + afterCount = q.MessageCount + } + } + if afterCount == 1 { + break + } + time.Sleep(200 * time.Millisecond) + } + if afterCount != 1 { + t.Errorf("post-drain message count for %s = %d, want 1 (the stuck scheduled record)", redirectQueue, afterCount) + } + + // Verify every drained record's body sha256 against the manifest (full + // coverage of all 511 retrievable records, not just a sample), and + // bucket by manifest origin queue to confirm the expected composition: + // 5 plain + 5 props + 500 paged + 1 large = 511. + var plainCount, propsCount, pagedCount, largeCount int + var propsSample *amqp.Message + for i, rec := range sink.records { + var am amqp.Message + if err := am.UnmarshalBinary(rec.AMQP); err != nil { + t.Fatalf("record %d: unmarshal AMQP: %v", i, err) + } + body := am.GetData() + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + origin, ok := bodyOrigin[hash] + if !ok { + t.Errorf("record %d body sha256 %s (len %d) not found in manifest", i, hash, len(body)) + continue + } + switch origin { + case "salvage.plain": + plainCount++ + case "salvage.props": + propsCount++ + if propsSample == nil { + propsSample = &am + } + case "salvage.paged": + pagedCount++ + case "salvage.large": + largeCount++ + default: + t.Errorf("record %d body sha256 %s belongs to unexpected manifest queue %s", i, hash, origin) + } + } + if plainCount != 5 { + t.Errorf("plain-origin records = %d, want 5", plainCount) + } + if propsCount != 5 { + t.Errorf("props-origin records = %d, want 5", propsCount) + } + if pagedCount != 500 { + t.Errorf("paged-origin records = %d, want 500", pagedCount) + } + if largeCount != 1 { + t.Errorf("large-origin records = %d, want 1", largeCount) + } + + // Property round-trip: confirm a props-origin record carries its + // original application properties (attempt=1, region=eu per + // manifest.json's salvage.props entries) byte-for-byte through + // salvage -> store -> redeliver -> broker. + if propsSample == nil { + t.Fatal("no props-origin record found to verify properties on") + } + if got := fmt.Sprint(propsSample.ApplicationProperties["region"]); got != "eu" { + t.Errorf("props record region = %q, want %q", got, "eu") + } + if got := fmt.Sprint(propsSample.ApplicationProperties["attempt"]); got != "1" { + t.Errorf("props record attempt = %q, want %q", got, "1") + } +} diff --git a/internal/cli/salvage_test.go b/internal/cli/salvage_test.go new file mode 100644 index 0000000..a47d419 --- /dev/null +++ b/internal/cli/salvage_test.go @@ -0,0 +1,698 @@ +package cli + +import ( + "archive/tar" + "compress/gzip" + "encoding/binary" + "io" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + + "github.com/martikan/artemisctl/internal/store" +) + +// salvageFixtureDir extracts internal/journal's committed +// artemis-2.42-data.tar.gz fixture into t.TempDir() and returns the +// extracted data/ path (containing bindings/journal/large-messages/paging). +// internal/journal's own fixtureDir helper (testdata_test.go) is unexported +// to that package, so this is a small local copy of the same extraction +// logic per the task brief. +func salvageFixtureDir(t *testing.T) string { + t.Helper() + tarball := filepath.Join("..", "journal", "testdata", "artemis-2.42-data.tar.gz") + f, err := os.Open(tarball) + if err != nil { + if os.IsNotExist(err) { + t.Skipf("fixture %s missing; run `make fixtures` to harvest it", tarball) + } + t.Fatalf("open fixture: %v", err) + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("gunzip fixture: %v", err) + } + defer gz.Close() + + dst := t.TempDir() + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("read fixture tar: %v", err) + } + name := filepath.Clean(hdr.Name) + if strings.HasPrefix(name, "..") || filepath.IsAbs(name) { + t.Fatalf("fixture tar has unsafe path %q", hdr.Name) + } + path := filepath.Join(dst, name) + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("extract dir %s: %v", name, err) + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("extract parent of %s: %v", name, err) + } + out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + t.Fatalf("extract file %s: %v", name, err) + } + if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // trusted committed fixture + out.Close() + t.Fatalf("extract file %s: %v", name, err) + } + if err := out.Close(); err != nil { + t.Fatalf("close extracted %s: %v", name, err) + } + default: + t.Fatalf("fixture tar has unexpected entry type %d for %q", hdr.Typeflag, hdr.Name) + } + } + return filepath.Join(dst, "data") +} + +// writeEmptyJournalFile writes a syntactically valid but empty journal file +// (just the 16-byte header: int formatVersion=2, int userVersion=0, long +// fileID=1 -- format_notes.md section 1) at dir/activemq-data-1.amq, so +// ReadJournalDir accepts the file and yields zero records. +func writeEmptyJournalFile(t *testing.T, dir string) { + t.Helper() + var hdr [16]byte + binary.BigEndian.PutUint32(hdr[0:4], 2) // formatVersion + binary.BigEndian.PutUint32(hdr[4:8], 0) // userVersion + binary.BigEndian.PutUint64(hdr[8:16], 1) // fileID + path := filepath.Join(dir, "activemq-data-1.amq") + if err := os.WriteFile(path, hdr[:], 0o644); err != nil { + t.Fatalf("write empty journal file: %v", err) + } +} + +// runSalvage runs `artemisctl salvage ` via cobra's ExecuteC and +// returns captured combined stdout/stderr and the RunE error. +func runSalvage(t *testing.T, args ...string) (string, error) { + t.Helper() + return runCmdStdin(t, "", append([]string{"salvage"}, args...)...) +} + +// writeCorePagedEntry writes a single-entry page file at +// //1.page containing one Core-persister page +// entry: an 8-byte transactionID (value irrelevant -- read and discarded), +// a 1-byte largeMessageType of NONE (0), and a 1-byte persister id of Core +// (1) -- format_notes.md section 8's outer '{' size '}' framing wraps a body +// paging.go's decodePagedMessage classifies as persisterDecodeCore, which +// ReadPaging counts as a Skip (PagingDiag.CoreSkipped) without ever +// producing an exportable PagedMessage. +// +// This exists to build a zero-exported-records-but-with-a-skip fixture +// (finding I1's gating test) cheaply: a page file's framing is a simple +// size-prefixed block, unlike the message/bindings journal's much heavier +// framing (16-byte header, fileID echo, check-size), so it is far less work +// to hand-encode from scratch than an equivalent Core-skipped message-journal +// record would be. +func writeCorePagedEntry(t *testing.T, pagingDir string) { + t.Helper() + addrDir := filepath.Join(pagingDir, "salvage.corepaged") + if err := os.MkdirAll(addrDir, 0o755); err != nil { + t.Fatal(err) + } + + var body [10]byte + binary.BigEndian.PutUint64(body[0:8], 0) // transactionID: irrelevant, read and discarded + body[8] = 0 // largeMessageType = NONE + body[9] = 1 // persister id = Core (1) -> persisterDecodeCore -> CoreSkipped + + frame := make([]byte, 0, 1+4+len(body)+1) + frame = append(frame, '{') + var size [4]byte + binary.BigEndian.PutUint32(size[:], uint32(len(body))) + frame = append(frame, size[:]...) + frame = append(frame, body[:]...) + frame = append(frame, '}') + + if err := os.WriteFile(filepath.Join(addrDir, "1.page"), frame, 0o644); err != nil { + t.Fatal(err) + } +} + +func TestSalvageFixtureSuccess(t *testing.T) { + dir := salvageFixtureDir(t) + out := filepath.Join(t.TempDir(), "rescue.artx") + + stdout, err := runSalvage(t, "--data", dir, "--out", out) + if err != nil { + t.Fatalf("salvage: %v\noutput:\n%s", err, stdout) + } + + // Full-block equality, not per-line Contains soup: this pins line ORDER + // (alphabetical by queue name), the exact column alignment (width = + // longest queue name "salvage.scheduled" + 2), the "(largest N)" + // annotation landing on the one queue attributed the large message, and + // that no "skipped:" section is emitted for this zero-skip fixture. + // Golden value captured from a verified-correct run of this exact + // fixture (artemis-2.42-data.tar.gz, 512 total records across 5 queues) + // and re-verified by `go test -run TestSalvageFixtureSuccess -v`. + wantStdout := "salvaged 512 messages to " + out + "\n" + + " salvage.large 1 (largest 300.1 KiB)\n" + + " salvage.paged 500\n" + + " salvage.plain 5\n" + + " salvage.props 5\n" + + " salvage.scheduled 1\n" + if stdout != wantStdout { + t.Errorf("stdout mismatch:\n got: %q\n want: %q", stdout, wantStdout) + } + + fi, statErr := os.Stat(out) + if statErr != nil { + t.Fatalf("stat --out: %v", statErr) + } + if fi.Size() == 0 { + t.Fatal("--out is empty") + } + if _, statErr := os.Stat(out + ".partial"); !os.IsNotExist(statErr) { + t.Errorf(".partial file left behind: %v", statErr) + } + + rd, err := store.OpenReader(out) + if err != nil { + t.Fatalf("OpenReader: %v", err) + } + defer rd.Close() + total := 0 + for { + _, _, err := rd.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Next: %v", err) + } + total++ + } + if total != 512 { + t.Errorf("store has %d records, want 512", total) + } +} + +func TestSalvageBindingsAndJournalWithoutData(t *testing.T) { + dir := salvageFixtureDir(t) + out := filepath.Join(t.TempDir(), "rescue.artx") + + stdout, err := runSalvage(t, + "--bindings", filepath.Join(dir, "bindings"), + "--journal", filepath.Join(dir, "journal"), + "--large-messages", filepath.Join(dir, "large-messages"), + "--paging", filepath.Join(dir, "paging"), + "--out", out, + ) + if err != nil { + t.Fatalf("salvage: %v\noutput:\n%s", err, stdout) + } + if !strings.Contains(stdout, "salvaged 512 messages to "+out) { + t.Errorf("stdout missing headline; got:\n%s", stdout) + } +} + +func TestSalvageMissingDataFlagCombo(t *testing.T) { + out := filepath.Join(t.TempDir(), "rescue.artx") + + // Neither --data nor the full --bindings+--journal pair: rejected before + // touching the filesystem. + _, err := runSalvage(t, "--journal", "/nonexistent/journal", "--out", out) + if err == nil { + t.Fatal("want error when --data is absent and --bindings is not set") + } + if !strings.Contains(err.Error(), "--data is required") { + t.Errorf("error = %v, want mention of --data requirement", err) + } +} + +func TestSalvageZeroRecords(t *testing.T) { + dataDir := t.TempDir() + bindings := filepath.Join(dataDir, "bindings") + journalDir := filepath.Join(dataDir, "journal") + if err := os.MkdirAll(bindings, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(journalDir, 0o755); err != nil { + t.Fatal(err) + } + writeEmptyJournalFile(t, journalDir) + + out := filepath.Join(t.TempDir(), "rescue.artx") + stdout, err := runSalvage(t, "--data", dataDir, "--out", out) + if err != nil { + t.Fatalf("salvage: %v\noutput:\n%s", err, stdout) + } + if !strings.Contains(stdout, "salvaged 0 messages") { + t.Errorf("stdout missing zero-record headline; got:\n%s", stdout) + } + if _, statErr := os.Stat(out); !os.IsNotExist(statErr) { + t.Fatalf("--out must not exist after a zero-record run, stat err = %v", statErr) + } + if _, statErr := os.Stat(out + ".partial"); !os.IsNotExist(statErr) { + t.Fatalf(".partial must not be left behind, stat err = %v", statErr) + } +} + +// TestSalvageZeroRecordsWithSkipsGated pins finding I1: a run that salvages +// zero records must still be gated on skips (and corruption) exactly like a +// non-empty run -- exiting 0 on "nothing recovered, but something was lost" +// is the worst case for a scripted recovery, since it is silently +// indistinguishable from "genuinely nothing here to salvage". The journal +// and bindings dirs are empty (zero exported records); the paging dir holds +// one Core-protocol page entry (writeCorePagedEntry), which is a Skip that +// never contributes to PerQueue. +func TestSalvageZeroRecordsWithSkipsGated(t *testing.T) { + dataDir := t.TempDir() + bindings := filepath.Join(dataDir, "bindings") + journalDir := filepath.Join(dataDir, "journal") + pagingDir := filepath.Join(dataDir, "paging") + if err := os.MkdirAll(bindings, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(journalDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(pagingDir, 0o755); err != nil { + t.Fatal(err) + } + writeEmptyJournalFile(t, journalDir) + writeCorePagedEntry(t, pagingDir) + + t.Run("without allow-skips fails", func(t *testing.T) { + out := filepath.Join(t.TempDir(), "rescue.artx") + stdout, err := runSalvage(t, "--data", dataDir, "--out", out) + if err == nil { + t.Fatalf("want error for a zero-record run with skips present; output:\n%s", stdout) + } + if !strings.Contains(err.Error(), "skips or corruption present") { + t.Errorf("error = %v, want it to mention skips/corruption", err) + } + if !strings.Contains(stdout, "salvaged 0 messages") { + t.Errorf("stdout missing zero-record headline; got:\n%s", stdout) + } + if !strings.Contains(stdout, "skipped:") { + t.Errorf("stdout missing skipped section; got:\n%s", stdout) + } + if _, statErr := os.Stat(out); !os.IsNotExist(statErr) { + t.Errorf("--out must not be created for a zero-record run, stat err = %v", statErr) + } + if _, statErr := os.Stat(out + ".partial"); !os.IsNotExist(statErr) { + t.Errorf(".partial must not be left behind, stat err = %v", statErr) + } + }) + + t.Run("with allow-skips succeeds", func(t *testing.T) { + out := filepath.Join(t.TempDir(), "rescue.artx") + stdout, err := runSalvage(t, "--data", dataDir, "--out", out, "--allow-skips") + if err != nil { + t.Fatalf("salvage with --allow-skips: %v\noutput:\n%s", err, stdout) + } + if !strings.Contains(stdout, "salvaged 0 messages") { + t.Errorf("stdout missing zero-record headline; got:\n%s", stdout) + } + if !strings.Contains(stdout, "skipped:") { + t.Errorf("stdout missing skipped section; got:\n%s", stdout) + } + if _, statErr := os.Stat(out); !os.IsNotExist(statErr) { + t.Errorf("--out must still never be created for a zero-record run, even with --allow-skips, stat err = %v", statErr) + } + }) +} + +func TestSalvageMissingJournalDir(t *testing.T) { + dataDir := t.TempDir() + bindings := filepath.Join(dataDir, "bindings") + if err := os.MkdirAll(bindings, 0o755); err != nil { + t.Fatal(err) + } + // journal/ deliberately not created. + + out := filepath.Join(t.TempDir(), "rescue.artx") + _, err := runSalvage(t, "--data", dataDir, "--out", out) + if err == nil { + t.Fatal("want error for missing journal dir") + } + if !strings.Contains(err.Error(), "journal") { + t.Errorf("error = %v, want it to mention the journal dir", err) + } + if _, statErr := os.Stat(out); !os.IsNotExist(statErr) { + t.Fatal("--out must not be created when validation fails") + } +} + +// TestSalvageSkipsGating exercises both sides of the --allow-skips gate using +// a deliberately-missing --large-messages dir: the fixture's one large +// message then becomes an unrecoverable skip (its body file is unreachable), +// while everything else still salvages fine. +func TestSalvageSkipsGating(t *testing.T) { + dir := salvageFixtureDir(t) + missingLarge := filepath.Join(t.TempDir(), "no-such-large-messages-dir") + + t.Run("without allow-skips fails", func(t *testing.T) { + out := filepath.Join(t.TempDir(), "rescue.artx") + stdout, err := runSalvage(t, + "--bindings", filepath.Join(dir, "bindings"), + "--journal", filepath.Join(dir, "journal"), + "--large-messages", missingLarge, + "--paging", filepath.Join(dir, "paging"), + "--out", out, + ) + if err == nil { + t.Fatalf("want error when skips are present without --allow-skips; output:\n%s", stdout) + } + if !strings.Contains(err.Error(), "skips or corruption present") { + t.Errorf("error = %v, want it to mention skips", err) + } + if !strings.Contains(stdout, "skipped:") { + t.Errorf("stdout missing skipped section; got:\n%s", stdout) + } + // A partial (but real) result is still saved -- a recovery tool must + // not throw away what it *did* manage to salvage. + if fi, statErr := os.Stat(out); statErr != nil || fi.Size() == 0 { + t.Errorf("--out should still be written despite the skip-gated exit: stat err=%v", statErr) + } + }) + + t.Run("with allow-skips succeeds", func(t *testing.T) { + out := filepath.Join(t.TempDir(), "rescue.artx") + stdout, err := runSalvage(t, + "--bindings", filepath.Join(dir, "bindings"), + "--journal", filepath.Join(dir, "journal"), + "--large-messages", missingLarge, + "--paging", filepath.Join(dir, "paging"), + "--out", out, + "--allow-skips", + ) + if err != nil { + t.Fatalf("salvage with --allow-skips: %v\noutput:\n%s", err, stdout) + } + if !strings.Contains(stdout, "skipped:") { + t.Errorf("stdout missing skipped section; got:\n%s", stdout) + } + }) +} + +// TestSalvageCorruptJournalGatesExitAndPrintsDiagnostics pins finding C1 +// end-to-end through the actual CLI binary: a corrupted message-journal +// record must show up as a "diagnostics:" section in stdout (not silently +// swallowed) and must gate the exit code exactly like a skip, honoring +// --allow-skips the same way. +// +// The corrupted byte is the last byte of a record's trailing check-size int +// (the same technique internal/journal's +// TestReadJournalDirCorruptRecordResyncs uses against this exact fixture +// file) at a fixture-verified, deterministic offset: the fixture tarball is +// an immutable committed file, so activemq-data-1.amq's second record (an +// UPDATE_RECORD/ADD_REF for message id 38) always starts at byte 273 and +// ends at byte 304 in a fresh extraction -- offset 303 is the last byte of +// its check-size. This offset is pinned as a literal (rather than located +// dynamically via internal/journal's unexported scanCleanSpans/parseRecord, +// which this package cannot import) because it is deterministic against the +// committed fixture; internal/journal/file_test.go independently re-derives +// and asserts the same offset every run, so a fixture change that moved it +// would already fail there first. +func TestSalvageCorruptJournalGatesExitAndPrintsDiagnostics(t *testing.T) { + dir := salvageFixtureDir(t) + journalFile := filepath.Join(dir, "journal", "activemq-data-1.amq") + + data, err := os.ReadFile(journalFile) + if err != nil { + t.Fatalf("read journal file: %v", err) + } + const corruptOffset = 303 // fixture-verified: last byte of record[1]'s trailing check-size int + data[corruptOffset] ^= 0xFF + if err := os.WriteFile(journalFile, data, 0o644); err != nil { + t.Fatalf("write corrupted journal file: %v", err) + } + + t.Run("without allow-skips fails and reports", func(t *testing.T) { + out := filepath.Join(t.TempDir(), "rescue.artx") + stdout, err := runSalvage(t, "--data", dir, "--out", out) + if err == nil { + t.Fatalf("want error for a corrupt journal record without --allow-skips; output:\n%s", stdout) + } + if !strings.Contains(err.Error(), "skips or corruption present") { + t.Errorf("error = %v, want it to mention skips/corruption", err) + } + if !strings.Contains(stdout, "diagnostics:") { + t.Errorf("stdout missing diagnostics section; got:\n%s", stdout) + } + if !strings.Contains(stdout, "check-size mismatch") { + t.Errorf("stdout diagnostics missing the check-size-mismatch incident; got:\n%s", stdout) + } + // A partial (but real) result is still saved, same contract as a + // skip-gated exit. + if fi, statErr := os.Stat(out); statErr != nil || fi.Size() == 0 { + t.Errorf("--out should still be written despite the corruption-gated exit: stat err=%v", statErr) + } + }) + + t.Run("with allow-skips succeeds and still reports", func(t *testing.T) { + out := filepath.Join(t.TempDir(), "rescue.artx") + stdout, err := runSalvage(t, "--data", dir, "--out", out, "--allow-skips") + if err != nil { + t.Fatalf("salvage with --allow-skips: %v\noutput:\n%s", err, stdout) + } + if !strings.Contains(stdout, "diagnostics:") { + t.Errorf("stdout missing diagnostics section; got:\n%s", stdout) + } + }) +} + +func TestSalvageOutRefusedWhenExisting(t *testing.T) { + dir := salvageFixtureDir(t) + out := filepath.Join(t.TempDir(), "rescue.artx") + if err := os.WriteFile(out, []byte("not empty"), 0o600); err != nil { + t.Fatal(err) + } + + _, err := runSalvage(t, "--data", dir, "--out", out) + if err == nil { + t.Fatal("want error when --out already exists and is non-empty") + } + if !strings.Contains(err.Error(), "already exists") { + t.Errorf("error = %v, want already-exists message", err) + } + if _, statErr := os.Stat(out + ".partial"); !os.IsNotExist(statErr) { + t.Fatal("must not start writing a .partial file when --out is refused up front") + } + got, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if string(got) != "not empty" { + t.Fatal("existing --out content must be left untouched") + } +} + +// TestSalvageStalePartialIsConsumedNotRefused pins finding I3: a killed +// prior run can leave a non-empty .partial behind (store.NewWriter +// itself never got the chance to finish writing it, let alone rename it to +// --out). Before this fix, store.NewWriter's own non-empty-file refusal +// (there to protect a REAL store, i.e. --out, from accidental truncation) +// fired against that leftover .partial too, permanently blocking every +// re-run with a misleading "avoid overwriting drained data" error -- +// misleading because .partial is never a source of truth to begin with. The +// fix removes any stale .partial immediately before opening it for this +// run's write, leaving the separate, real --out check (a different code +// path, still exercised by TestSalvageOutRefusedWhenExisting) untouched. +func TestSalvageStalePartialIsConsumedNotRefused(t *testing.T) { + dir := salvageFixtureDir(t) + out := filepath.Join(t.TempDir(), "rescue.artx") + + // Simulate a killed prior run: a non-empty .partial with garbage content, + // no matching --out. + if err := os.WriteFile(out+".partial", []byte("leftover from a killed run"), 0o600); err != nil { + t.Fatal(err) + } + + stdout, err := runSalvage(t, "--data", dir, "--out", out) + if err != nil { + t.Fatalf("salvage: %v\noutput:\n%s", err, stdout) + } + if !strings.Contains(stdout, "salvaged 512 messages to "+out) { + t.Errorf("stdout missing success headline; got:\n%s", stdout) + } + + fi, statErr := os.Stat(out) + if statErr != nil || fi.Size() == 0 { + t.Fatalf("--out should be written normally, stat err=%v", statErr) + } + if _, statErr := os.Stat(out + ".partial"); !os.IsNotExist(statErr) { + t.Errorf(".partial should be consumed (renamed away) by a successful run, stat err = %v", statErr) + } + + rd, err := store.OpenReader(out) + if err != nil { + t.Fatalf("OpenReader: %v", err) + } + defer rd.Close() + total := 0 + for { + _, _, err := rd.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Next: %v", err) + } + total++ + } + if total != 512 { + t.Errorf("store has %d records, want 512 (the stale .partial's garbage content must not have survived into --out)", total) + } +} + +func TestSalvageOutRequired(t *testing.T) { + dir := salvageFixtureDir(t) + _, err := runSalvage(t, "--data", dir) + if err == nil { + t.Fatal("want error when --out is not given") + } +} + +// holdFlock opens path and takes a non-blocking exclusive flock on it, +// returning a cleanup func that unlocks and closes it. flock locks attach to +// the *open file description*, not the process or the path, so a second, +// independent os.OpenFile+syscall.Flock in this same process -- exactly +// what checkLiveBroker does -- gets EWOULDBLOCK against this one. No helper +// process is required to exercise the "lock held" path. +func holdFlock(t *testing.T, path string) func() { + t.Helper() + f, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + t.Fatalf("open %s to hold flock: %v", path, err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + f.Close() + t.Fatalf("acquire test flock on %s: %v", path, err) + } + return func() { + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + _ = f.Close() + } +} + +// TestSalvageCheckLiveBrokerLockHeld covers the primary, fixture-verified location +// (/server.lock): with the lock held by (this same process, a +// second independent fd) checkLiveBroker must refuse. +func TestSalvageCheckLiveBrokerLockHeld(t *testing.T) { + journalDir := t.TempDir() + lockPath := filepath.Join(journalDir, "server.lock") + if err := os.WriteFile(lockPath, nil, 0o644); err != nil { + t.Fatal(err) + } + release := holdFlock(t, lockPath) + defer release() + + err := checkLiveBroker(journalDir, "") + if err == nil { + t.Fatal("want error when server.lock is held") + } + if !strings.Contains(err.Error(), "broker appears to be running") { + t.Errorf("error = %v, want the live-broker message", err) + } +} + +// TestSalvageCheckLiveBrokerLockFileExistsButUnheld exercises the direct-call path +// explicitly: a server.lock file is present but nobody holds it, so the +// guard must pass (the fixture success tests exercise this implicitly -- +// their journal dirs carry no server.lock at all -- this pins the case +// where the file exists but is free). +func TestSalvageCheckLiveBrokerLockFileExistsButUnheld(t *testing.T) { + journalDir := t.TempDir() + lockPath := filepath.Join(journalDir, "server.lock") + if err := os.WriteFile(lockPath, nil, 0o644); err != nil { + t.Fatal(err) + } + if err := checkLiveBroker(journalDir, ""); err != nil { + t.Fatalf("checkLiveBroker = %v, want nil (lock file present but unheld)", err) + } +} + +// TestSalvageCheckLiveBrokerNoLockAnywhere covers the case where none of the +// candidate paths exist: nothing to check against, so salvage proceeds. +func TestSalvageCheckLiveBrokerNoLockAnywhere(t *testing.T) { + dataDir := t.TempDir() + journalDir := filepath.Join(dataDir, "journal") + if err := os.MkdirAll(journalDir, 0o755); err != nil { + t.Fatal(err) + } + if err := checkLiveBroker(journalDir, dataDir); err != nil { + t.Fatalf("checkLiveBroker = %v, want nil (no server.lock anywhere)", err) + } +} + +// TestSalvageCheckLiveBrokerInstanceRootFallback pins finding-1's fix: the instance +// root fallback (filepath.Dir(dataDir)/server.lock) is only reachable when +// dataDir is passed in, which it wasn't before this fix. A standard broker +// layout is /data/journal alongside /server.lock. +func TestSalvageCheckLiveBrokerInstanceRootFallback(t *testing.T) { + instanceRoot := t.TempDir() + dataDir := filepath.Join(instanceRoot, "data") + journalDir := filepath.Join(dataDir, "journal") + if err := os.MkdirAll(journalDir, 0o755); err != nil { + t.Fatal(err) + } + lockPath := filepath.Join(instanceRoot, "server.lock") + if err := os.WriteFile(lockPath, nil, 0o644); err != nil { + t.Fatal(err) + } + release := holdFlock(t, lockPath) + defer release() + + err := checkLiveBroker(journalDir, dataDir) + if err == nil { + t.Fatal("want error when the instance-root server.lock is held") + } + if !strings.Contains(err.Error(), "broker appears to be running") { + t.Errorf("error = %v, want the live-broker message", err) + } +} + +// TestSalvageForceBypassesLiveBrokerGuard drives the guard through the full +// `salvage` command: without --force a held journal-dir server.lock must +// refuse the run; with --force the command must proceed past the guard (it +// may still fail later for unrelated reasons against this minimal fixture, +// but that failure must not be the live-broker error). +func TestSalvageForceBypassesLiveBrokerGuard(t *testing.T) { + dataDir := t.TempDir() + bindings := filepath.Join(dataDir, "bindings") + journalDir := filepath.Join(dataDir, "journal") + if err := os.MkdirAll(bindings, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(journalDir, 0o755); err != nil { + t.Fatal(err) + } + writeEmptyJournalFile(t, journalDir) + + lockPath := filepath.Join(journalDir, "server.lock") + if err := os.WriteFile(lockPath, nil, 0o644); err != nil { + t.Fatal(err) + } + release := holdFlock(t, lockPath) + defer release() + + out := filepath.Join(t.TempDir(), "rescue.artx") + if _, err := runSalvage(t, "--data", dataDir, "--out", out); err == nil || + !strings.Contains(err.Error(), "broker appears to be running") { + t.Fatalf("want live-broker guard error without --force, got: %v", err) + } + + out2 := filepath.Join(t.TempDir(), "rescue.artx") + _, err := runSalvage(t, "--data", dataDir, "--out", out2, "--force") + if err != nil && strings.Contains(err.Error(), "broker appears to be running") { + t.Fatalf("--force should bypass the live-broker guard, got: %v", err) + } +} diff --git a/internal/cli/status.go b/internal/cli/status.go new file mode 100644 index 0000000..16ea890 --- /dev/null +++ b/internal/cli/status.go @@ -0,0 +1,36 @@ +package cli + +import ( + "context" + "fmt" + "text/tabwriter" + + "github.com/martikan/artemisctl/internal/broker" + "github.com/spf13/cobra" +) + +func newStatusCmd() *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "List queues and message counts (descending)", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx, cancel := connectCtx(cmd, context.Background()) + defer cancel() + c, err := broker.Connect(ctx, connProps(cmd)) + if err != nil { + return err + } + defer c.Close(ctx) + stats, err := c.ListQueues(ctx) + if err != nil { + return err + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "QUEUE NAME\tMESSAGE COUNT") + for _, s := range stats { + fmt.Fprintf(w, "%s\t%d\n", s.Name, s.MessageCount) + } + return w.Flush() + }, + } +} diff --git a/internal/cli/testhelpers_test.go b/internal/cli/testhelpers_test.go new file mode 100644 index 0000000..331d1c7 --- /dev/null +++ b/internal/cli/testhelpers_test.go @@ -0,0 +1,86 @@ +// internal/cli/testhelpers_test.go +package cli + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/Azure/go-amqp" + "github.com/martikan/artemisctl/internal/broker" + "github.com/martikan/artemisctl/internal/brokertest" + "github.com/martikan/artemisctl/internal/store" +) + +// startArtemisForCLI returns connection props for the shared integration broker, +// resetting it to a clean slate first. +func startArtemisForCLI(t *testing.T) broker.ConnectionProps { + t.Helper() + sc := brokertest.Shared(t) + props := broker.ConnectionProps{URL: sc.URL, Username: sc.Username, Password: sc.Password} + resetBrokerForCLI(t, props) + return props +} + +// discardSink drops every drained record; used only to purge queues. +type discardSink struct{} + +func (discardSink) Append(store.Record) error { return nil } +func (discardSink) Sync() error { return nil } + +// resetBrokerForCLI empties the shared broker before a CLI integration test, +// mirroring the broker package's resetBroker over the exported client API. +func resetBrokerForCLI(t *testing.T, props broker.ConnectionProps) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + c, err := broker.Connect(ctx, props) + if err != nil { + t.Fatalf("reset connect: %v", err) + } + defer c.Close(ctx) + // Lift any leftover cordon (see resetBroker in the broker package). + _ = c.Uncordon(ctx, brokertest.PermissiveWildcardSettings) + qs, err := c.ListQueues(ctx) + if err != nil { + t.Fatalf("reset list queues: %v", err) + } + for _, q := range qs { + if _, err := c.DrainQueue(ctx, q.Name, discardSink{}, 500*time.Millisecond, 200); err != nil { + t.Fatalf("reset drain %s: %v", q.Name, err) + } + } +} + +// seedQueue sends bodies to queue over a standalone AMQP connection. +// TargetCapabilities: []string{"queue"} tells Artemis to route this as an +// anycast queue rather than the default multicast address, so the queue is +// auto-created and actually holds the messages for export to drain. +func seedQueue(t *testing.T, props broker.ConnectionProps, queue string, bodies []string) { + t.Helper() + ctx := context.Background() + conn, err := amqp.Dial(ctx, "amqp://"+props.Username+":"+props.Password+"@"+props.URL, nil) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + sess, err := conn.NewSession(ctx, nil) + if err != nil { + t.Fatal(err) + } + sender, err := sess.NewSender(ctx, queue, &amqp.SenderOptions{TargetCapabilities: []string{"queue"}}) + if err != nil { + t.Fatal(err) + } + defer sender.Close(ctx) + for i, b := range bodies { + msg := amqp.NewMessage([]byte(b)) + // Real producers stamp an AMQP message-id; browse reports it as the ID + // column and BrowseMessage matches on it, so give each a unique one. + msg.Properties = &amqp.MessageProperties{MessageID: fmt.Sprintf("%s-%d-%s", queue, i, b)} + if err := sender.Send(ctx, msg, nil); err != nil { + t.Fatal(err) + } + } +} diff --git a/internal/cli/uncordon.go b/internal/cli/uncordon.go new file mode 100644 index 0000000..6f00a33 --- /dev/null +++ b/internal/cli/uncordon.go @@ -0,0 +1,60 @@ +package cli + +import ( + "context" + "fmt" + "os" + + "github.com/martikan/artemisctl/internal/broker" + "github.com/spf13/cobra" +) + +func newUncordonCmd() *cobra.Command { + var stateFile string + var forceRemove bool + + cmd := &cobra.Command{ + Use: "uncordon", + Short: "Lift a cordon, restoring the pre-cordon settings", + Long: "uncordon reverses cordon by restoring the settings saved in the state " + + "file. With --force-remove (or when no state file exists) it instead " + + "removes the wildcard settings entry, reverting to the broker's defaults.", + RunE: func(cmd *cobra.Command, _ []string) error { + ctx, cancel := connectCtx(cmd, context.Background()) + defer cancel() + c, err := broker.Connect(ctx, connProps(cmd)) + if err != nil { + return err + } + defer c.Close(context.Background()) + + state, readErr := readCordonState(stateFile) + if readErr != nil || forceRemove { + if readErr != nil && !forceRemove { + if os.IsNotExist(readErr) { + return fmt.Errorf("no state file at %s; re-run with --force-remove to clear the wildcard settings entry", stateFile) + } + return fmt.Errorf("read state file %s: %w", stateFile, readErr) + } + // --force-remove path: no saved state to restore. + if err := c.UncordonRemove(ctx); err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), "cordon lifted (wildcard settings entry removed)") + return nil + } + + if err := c.Uncordon(ctx, state.SavedSettings); err != nil { + return err + } + if err := os.Remove(stateFile); err != nil && !os.IsNotExist(err) { + fmt.Fprintf(cmd.OutOrStdout(), "warning: cordon lifted but could not remove state file %s: %v\n", stateFile, err) + } + fmt.Fprintf(cmd.OutOrStdout(), "cordon lifted; settings restored from %s\n", stateFile) + return nil + }, + } + cmd.Flags().StringVar(&stateFile, "state-file", defaultCordonState, "path to the pre-cordon settings written by cordon") + cmd.Flags().BoolVar(&forceRemove, "force-remove", false, "remove the wildcard settings entry instead of restoring saved state") + return cmd +} diff --git a/internal/journal/bindings.go b/internal/journal/bindings.go new file mode 100644 index 0000000..5e1c34e --- /dev/null +++ b/internal/journal/bindings.go @@ -0,0 +1,76 @@ +package journal + +import "fmt" + +// ReadQueueBindings replays the bindings journal and returns queueID → queue +// name for every surviving QUEUE_BINDING_RECORD. Other bindings journal +// record types (address bindings, queue status, address settings, security +// settings, diverts, bridges, …, see format_notes.md section 4) share the +// same record-ID namespace and file set but are ignored here: this reader +// only needs the queueID → name/address mapping to label salvaged messages. +// +// Records are replayed through Replayer so transaction semantics and +// DELETE_RECORD apply exactly as they do for the message journal (Task 5): +// a queue binding that was deleted (queue removed) before the broker died +// does not survive into the returned map. +// +// A single binding record whose body fails to decode (e.g. a garbled +// length-prefix byte) does NOT abort the run: that one binding is skipped +// and reported as a corruption-class FileDiag (id + reason), and every other +// binding still decodes normally. The messages that would have resolved +// through the skipped binding still export fine -- they simply fall back to +// the synthetic unknown-queue- name (salvage.go's queueName), exactly as +// they would for a queue whose binding legitimately never existed. Aborting +// the entire salvage over one damaged bindings-journal record would be a far +// worse outcome than that fallback (spec §1: "a recovery tool must not +// silently lose messages" -- losing everything over one bad record is its +// own kind of silent loss). +func ReadQueueBindings(bindingsDir string) (map[int64]string, []FileDiag, error) { + replayer := NewReplayer() + diags, err := ReadJournalDir(bindingsDir, "activemq-bindings", "bindings", func(r RawRecord) error { + return replayer.Feed(r) + }) + if err != nil { + return nil, diags, err + } + + survivors, _ := replayer.Resolve() + + out := make(map[int64]string, len(survivors)) + for _, sv := range survivors { + if sv.UserType != QueueBindingRecord { + continue + } + + name, address, err := decodeQueueBinding(sv.Body) + if err != nil { + diags = append(diags, FileDiag{ + Path: bindingsDir, + Reason: fmt.Sprintf("decode queue binding id %d: %v", sv.ID, err), + Corrupt: true, + }) + continue + } + _ = address // read for framing correctness only; not surfaced by this API (brief: name/address for a debug log) + + out[sv.ID] = name + } + return out, diags, nil +} + +// decodeQueueBinding decodes the stable prefix of +// PersistentQueueBindingEncoding.decode (format_notes.md section 9's +// "Supporting codec field orders"): SimpleString queueName, SimpleString +// address. The remaining fields (nullable filter string, nullable user +// metadata, autoCreated boolean, and a versioned tail of routing-type/ +// max-consumers/purge flags) are intentionally left unread -- the tail's +// shape is version-dependent and salvage does not need it. +func decodeQueueBinding(body []byte) (name, address string, err error) { + r := newReader(body) + name = r.simpleString() + address = r.simpleString() + if r.err() != nil { + return "", "", r.err() + } + return name, address, nil +} diff --git a/internal/journal/bindings_test.go b/internal/journal/bindings_test.go new file mode 100644 index 0000000..3d7903b --- /dev/null +++ b/internal/journal/bindings_test.go @@ -0,0 +1,133 @@ +package journal + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReadQueueBindings(t *testing.T) { + dir := fixtureDir(t) + bindingsDir := filepath.Join(dir, "bindings") + + got, diags, err := ReadQueueBindings(bindingsDir) + if err != nil { + t.Fatalf("ReadQueueBindings: %v", err) + } + if len(diags) != 0 { + t.Fatalf("want 0 diags over the clean fixture, got %d: %+v", len(diags), diags) + } + + names := make(map[string]bool, len(got)) + for _, name := range got { + names[name] = true + } + + want := []string{ + "salvage.plain", + "salvage.props", + "salvage.scheduled", + "salvage.large", + "salvage.paged", + "salvage.acked", + } + for _, q := range want { + if !names[q] { + t.Errorf("want queue %q present in bindings, got %v", q, got) + } + } +} + +func TestReadQueueBindingsIgnoresNonQueueBindingRecords(t *testing.T) { + dir := fixtureDir(t) + bindingsDir := filepath.Join(dir, "bindings") + + got, _, err := ReadQueueBindings(bindingsDir) + if err != nil { + t.Fatalf("ReadQueueBindings: %v", err) + } + + // Every value must be a plausible queue name, never empty -- a + // non-QUEUE_BINDING_RECORD accidentally decoded as one would typically + // produce an empty or garbage name. + for id, name := range got { + if name == "" { + t.Errorf("queue id %d decoded to empty name", id) + } + } +} + +// TestReadQueueBindingsCorruptBindingBodySkipsOneAndSurvives pins finding I2: +// a single binding record whose body fails to decode (e.g. a garbled +// length-prefix byte) must not abort ReadQueueBindings for the whole +// bindings journal -- only that one binding is skipped (reported as a +// corruption-class FileDiag), and every other binding still resolves +// normally. Reviewer's proof used the exact same technique: garbling one +// queueName length-prefix byte on binding id 3 killed the entire salvage run +// before this fix ("decode queue binding id 3: ... truncated at offset 4"). +func TestReadQueueBindingsCorruptBindingBodySkipsOneAndSurvives(t *testing.T) { + dir := fixtureDir(t) + srcPath := filepath.Join(dir, "bindings", "activemq-bindings-1.bindings") + data, err := os.ReadFile(srcPath) + if err != nil { + t.Fatalf("read fixture file: %v", err) + } + + // Locate QUEUE_BINDING_RECORD id=3's body and garble its first byte -- + // the high byte of the queueName SimpleString's 4-byte big-endian + // byte-length prefix (format_notes.md section 6) -- without touching the + // record's framing/checkSize (only body content changes, not any + // length field the framing layer itself checks). This makes + // decodeQueueBinding's SimpleString length read implausible (negative or + // wildly oversized), failing deep inside the body rather than at the + // framing layer -- exactly the class of damage this fix targets. + const bindingID3RecordStart = 117 // fixture-verified offset of id=3's ADD_RECORD_TX (activemq-bindings-1.bindings) + const fixedPrefixLen = 27 // type(1)+fileIDEcho(4)+compactCount(1)+txID(8)+recordID(8)+variableSize(4)+userType(1), precedes the body (id=3 is journaled inside a transaction, hence the extra txID field vs. a plain ADD_RECORD) + corruptOffset := bindingID3RecordStart + fixedPrefixLen + + corrupted := append([]byte(nil), data...) + corrupted[corruptOffset] ^= 0xFF + + dstDir := t.TempDir() + if err := os.WriteFile(filepath.Join(dstDir, "activemq-bindings-1.bindings"), corrupted, 0o644); err != nil { + t.Fatalf("write corrupted bindings file: %v", err) + } + // Copy the fixture's second bindings file unmodified so its bindings + // still participate normally alongside the corrupted file's survivors. + if data2, err := os.ReadFile(filepath.Join(dir, "bindings", "activemq-bindings-2.bindings")); err == nil { + if err := os.WriteFile(filepath.Join(dstDir, "activemq-bindings-2.bindings"), data2, 0o644); err != nil { + t.Fatalf("copy activemq-bindings-2.bindings: %v", err) + } + } + + got, diags, err := ReadQueueBindings(dstDir) + if err != nil { + t.Fatalf("ReadQueueBindings: %v (want the run to survive one garbled binding, not abort)", err) + } + + if name, stillPresent := got[3]; stillPresent { + t.Errorf("binding id 3 should have been skipped after body corruption, got %q", name) + } + // Every other binding in the fixture (fixture-verified ids from + // TestZZ-equivalent manual inspection of activemq-bindings-1.bindings) + // must survive untouched. + for _, id := range []int64{7, 13, 35, 45, 55, 61, 67, 82} { + if _, ok := got[id]; !ok { + t.Errorf("binding id %d missing after unrelated corruption; got %v", id, got) + } + } + + foundDiag := false + for _, d := range diags { + if strings.Contains(d.Reason, "decode queue binding id 3") { + foundDiag = true + if !d.Corrupt { + t.Errorf("diag for binding id 3 has Corrupt=false, want true (structural bindings-record damage): %+v", d) + } + } + } + if !foundDiag { + t.Errorf("diags missing entry for binding id 3: %+v", diags) + } +} diff --git a/internal/journal/core_payload.go b/internal/journal/core_payload.go new file mode 100644 index 0000000..c2adf8d --- /dev/null +++ b/internal/journal/core_payload.go @@ -0,0 +1,281 @@ +package journal + +import ( + "encoding/binary" + "errors" + "fmt" + "math" +) + +// Core payload store encoding. +// +// A decoded Core message (CorePayload) is serialized into an opaque byte blob +// that the .artx store carries verbatim (store.Record.CorePayload, Kind=Core) +// and broker/coreconvert reads back to build an amqp.Message for redelivery. +// The encoding is little-endian to match the store's own convention (the +// on-disk Artemis journal is big-endian; this is our format, not Artemis's). +// +// Property values preserve the Artemis TypedProperties value types (bool, +// byte, int16/32/64, float32/64, string, []byte) via a one-byte type tag so +// the AMQP conversion can map each to the right application-property type. + +// core payload value type tags (independent of Artemis DataConstants ids). +const ( + cpvBool byte = iota + cpvByte + cpvInt16 + cpvInt32 + cpvInt64 + cpvFloat32 + cpvFloat64 + cpvString + cpvBytes + cpvNull +) + +// coreWriter is a little-endian append buffer. +type coreWriter struct{ b []byte } + +func (w *coreWriter) u8(v byte) { w.b = append(w.b, v) } +func (w *coreWriter) u16(v int) { w.b = binary.LittleEndian.AppendUint16(w.b, uint16(v)) } +func (w *coreWriter) u32(v int) { w.b = binary.LittleEndian.AppendUint32(w.b, uint32(v)) } +func (w *coreWriter) i64(v int64) { w.b = binary.LittleEndian.AppendUint64(w.b, uint64(v)) } +func (w *coreWriter) str(s string) { + w.u32(len(s)) + w.b = append(w.b, s...) +} +func (w *coreWriter) blob(p []byte) { + w.u32(len(p)) + w.b = append(w.b, p...) +} + +// Encode serializes p into the store's opaque Core payload blob. +func (p *CorePayload) Encode() []byte { + w := &coreWriter{} + w.i64(p.MessageID) + w.str(p.Address) + w.u8(byte(len(p.UserID))) + w.b = append(w.b, p.UserID...) + w.u8(p.Type) + if p.Durable { + w.u8(1) + } else { + w.u8(0) + } + w.i64(p.Expiration) + w.i64(p.Timestamp) + w.u8(p.Priority) + if p.Large { + w.u8(1) + } else { + w.u8(0) + } + encodeCoreProps(w, p.Properties) + w.blob(p.Body) + return w.b +} + +func encodeCoreProps(w *coreWriter, props map[string]any) { + w.u32(len(props)) + for k, v := range props { + w.str(k) + encodeCoreValue(w, v) + } +} + +func encodeCoreValue(w *coreWriter, v any) { + switch val := v.(type) { + case nil: + w.u8(cpvNull) + case bool: + w.u8(cpvBool) + if val { + w.u8(1) + } else { + w.u8(0) + } + case byte: // uint8 + w.u8(cpvByte) + w.u8(val) + case int16: + w.u8(cpvInt16) + w.u16(int(uint16(val))) + case int32: + w.u8(cpvInt32) + w.u32(int(uint32(val))) + case int64: + w.u8(cpvInt64) + w.i64(val) + case float32: + w.u8(cpvFloat32) + w.u32(int(math.Float32bits(val))) + case float64: + w.u8(cpvFloat64) + w.i64(int64(math.Float64bits(val))) + case string: + w.u8(cpvString) + w.str(val) + case []byte: + w.u8(cpvBytes) + w.blob(val) + default: + // Unknown value type: store its string form (lossy but never fatal). + w.u8(cpvString) + w.str(fmt.Sprint(val)) + } +} + +// coreReader is a bounds-checked little-endian cursor. +type coreReader struct { + b []byte + off int + err error +} + +func (r *coreReader) need(n int) bool { + if r.err != nil { + return false + } + if n < 0 || r.off+n > len(r.b) { + r.err = errors.New("core payload: truncated") + return false + } + return true +} +func (r *coreReader) u8() byte { + if !r.need(1) { + return 0 + } + v := r.b[r.off] + r.off++ + return v +} +func (r *coreReader) u16() int { + if !r.need(2) { + return 0 + } + v := binary.LittleEndian.Uint16(r.b[r.off:]) + r.off += 2 + return int(v) +} +func (r *coreReader) u32() int { + if !r.need(4) { + return 0 + } + v := binary.LittleEndian.Uint32(r.b[r.off:]) + r.off += 4 + return int(v) +} +func (r *coreReader) i64() int64 { + if !r.need(8) { + return 0 + } + v := binary.LittleEndian.Uint64(r.b[r.off:]) + r.off += 8 + return int64(v) +} +func (r *coreReader) take(n int) []byte { + if n < 0 || !r.need(n) { + return nil + } + v := r.b[r.off : r.off+n] + r.off += n + return append([]byte(nil), v...) +} +func (r *coreReader) str() string { + n := r.u32() + return string(r.take(n)) +} + +// DecodeCorePayload parses a Core payload blob produced by CorePayload.Encode. +func DecodeCorePayload(b []byte) (*CorePayload, error) { + r := &coreReader{b: b} + p := &CorePayload{} + p.MessageID = r.i64() + p.Address = r.str() + uidLen := int(r.u8()) + if uidLen > 0 { + p.UserID = r.take(uidLen) + } + p.Type = r.u8() + p.Durable = r.u8() != 0 + p.Expiration = r.i64() + p.Timestamp = r.i64() + p.Priority = r.u8() + p.Large = r.u8() != 0 + p.Properties = decodeCoreProps(r) + bodyLen := r.u32() + p.Body = r.take(bodyLen) + if r.err != nil { + return nil, r.err + } + return p, nil +} + +func decodeCoreProps(r *coreReader) map[string]any { + n := r.u32() + if r.err != nil || n == 0 { + return nil + } + props := make(map[string]any, n) + for i := 0; i < n; i++ { + key := r.str() + props[key] = decodeCoreValue(r) + if r.err != nil { + return props + } + } + return props +} + +// CoreTextBody decodes a Core TEXT message body, which is encoded as a +// nullableSimpleString (format_notes.md §6). Returns ("", false) if the body +// is not a well-formed SimpleString (caller should fall back to raw bytes). +func CoreTextBody(body []byte) (string, bool) { + r := newReader(body) + s, ok := r.nullableSimpleString() + if r.err() != nil { + return "", false + } + return s, ok +} + +// CoreMapBody decodes a Core MAP message body, encoded as a TypedProperties +// block (format_notes.md §7). Returns (nil, false) on a malformed body. +func CoreMapBody(body []byte) (map[string]any, bool) { + r := newReader(body) + m := r.typedProperties() + if r.err() != nil { + return nil, false + } + return m, true +} + +func decodeCoreValue(r *coreReader) any { + switch r.u8() { + case cpvNull: + return nil + case cpvBool: + return r.u8() != 0 + case cpvByte: + return r.u8() + case cpvInt16: + return int16(r.u16()) + case cpvInt32: + return int32(r.u32()) + case cpvInt64: + return r.i64() + case cpvFloat32: + return math.Float32frombits(uint32(r.u32())) + case cpvFloat64: + return math.Float64frombits(uint64(r.i64())) + case cpvString: + return r.str() + case cpvBytes: + n := r.u32() + return r.take(n) + default: + r.err = errors.New("core payload: unknown value tag") + return nil + } +} diff --git a/internal/journal/core_payload_extra_test.go b/internal/journal/core_payload_extra_test.go new file mode 100644 index 0000000..70aebcd --- /dev/null +++ b/internal/journal/core_payload_extra_test.go @@ -0,0 +1,156 @@ +package journal + +import ( + "bytes" + "math" + "reflect" + "testing" +) + +// TestCorePayloadRoundTripAllValueTypes exercises every typed property value +// (encodeCoreValue/decodeCoreValue) plus the non-empty UserID / Large / Durable +// Encode branches, so a Core message's normalized fields survive the store blob. +func TestCorePayloadRoundTripAllValueTypes(t *testing.T) { + props := map[string]any{ + "n": nil, + "bt": true, + "bf": false, + "by": byte(0x7B), + "i16": int16(-1234), + "i32": int32(-123456), + "i64": int64(-123456789012), + "f32": float32(3.14), + "f64": float64(2.71828), + "s": "héllo", + "raw": []byte{0x01, 0x02, 0x03}, + "unk": uint32(9), // unhandled type -> stored as its string form "9" + } + orig := &CorePayload{ + MessageID: 42, + Address: "salvage.core", + UserID: bytes.Repeat([]byte{0xAB}, 16), + Type: CoreTypeBytes, + Durable: true, + Expiration: 1_700_000_000_123, + Timestamp: 1_699_999_999_000, + Priority: 9, + Large: true, + Properties: props, + Body: []byte("body-bytes"), + } + got, err := DecodeCorePayload(orig.Encode()) + if err != nil { + t.Fatalf("DecodeCorePayload: %v", err) + } + if got.MessageID != orig.MessageID || got.Address != orig.Address || + !bytes.Equal(got.UserID, orig.UserID) || got.Type != orig.Type || + !got.Durable || got.Expiration != orig.Expiration || got.Timestamp != orig.Timestamp || + got.Priority != orig.Priority || !got.Large || string(got.Body) != string(orig.Body) { + t.Fatalf("scalar mismatch:\n got=%+v\norig=%+v", got, orig) + } + want := map[string]any{ + "n": nil, "bt": true, "bf": false, "by": byte(0x7B), + "i16": int16(-1234), "i32": int32(-123456), "i64": int64(-123456789012), + "f32": float32(3.14), "f64": float64(2.71828), "s": "héllo", + "raw": []byte{0x01, 0x02, 0x03}, "unk": "9", + } + if !reflect.DeepEqual(got.Properties, want) { + t.Errorf("props mismatch:\n got=%#v\nwant=%#v", got.Properties, want) + } +} + +// TestCorePayloadRoundTripMinimal covers the Durable=false / Large=false / empty +// UserID / nil-Properties Encode branches. +func TestCorePayloadRoundTripMinimal(t *testing.T) { + orig := &CorePayload{MessageID: 1, Address: "q", Type: CoreTypeText, Body: []byte("x")} + got, err := DecodeCorePayload(orig.Encode()) + if err != nil { + t.Fatalf("DecodeCorePayload: %v", err) + } + if got.Durable || got.Large || len(got.UserID) != 0 || len(got.Properties) != 0 { + t.Errorf("minimal mismatch: %+v", got) + } +} + +func TestDecodeCorePayloadTruncated(t *testing.T) { + full := (&CorePayload{MessageID: 7, Address: "abc", Body: []byte("hello")}).Encode() + if _, err := DecodeCorePayload(full[:len(full)-3]); err == nil { + t.Fatalf("want truncation error, got nil") + } + if _, err := DecodeCorePayload(nil); err == nil { + t.Fatalf("want error on empty blob, got nil") + } +} + +func TestCoreTextBody(t *testing.T) { + var present bytes.Buffer + writeNullableSimpleStringBytes(&present, "core-hello", true) + if s, ok := CoreTextBody(present.Bytes()); !ok || s != "core-hello" { + t.Errorf("present: got (%q,%v), want (core-hello,true)", s, ok) + } + + var absent bytes.Buffer + writeNullableSimpleStringBytes(&absent, "", false) + if s, ok := CoreTextBody(absent.Bytes()); ok || s != "" { + t.Errorf("absent: got (%q,%v), want (\"\",false)", s, ok) + } + + if s, ok := CoreTextBody([]byte{0x01, 0xFF}); ok || s != "" { + t.Errorf("malformed: got (%q,%v), want (\"\",false)", s, ok) + } +} + +func TestCoreMapBody(t *testing.T) { + var buf bytes.Buffer + writeTypedPropsHeader(&buf, 2) + writeTypedPropKey(&buf, "kInt") + buf.WriteByte(6) + buf.Write(be32(99)) + writeTypedPropKey(&buf, "kString") + buf.WriteByte(10) + writeSimpleStringBytes(&buf, "v") + + m, ok := CoreMapBody(buf.Bytes()) + if !ok { + t.Fatalf("CoreMapBody ok=false, want true") + } + if m["kInt"] != int32(99) || m["kString"] != "v" { + t.Errorf("map = %#v, want {kInt:99, kString:v}", m) + } + + // malformed: NOT_NULL marker then an impossibly large count. + if _, ok := CoreMapBody([]byte{0x01, 0x7F, 0xFF, 0xFF, 0xFF}); ok { + t.Errorf("malformed CoreMapBody ok=true, want false") + } +} + +// TestDecodeCoreValueUnknownTag hits decodeCoreValue's default (unknown tag) +// branch via a hand-built props blob DecodeCorePayload cannot otherwise produce. +func TestDecodeCoreValueUnknownTag(t *testing.T) { + w := &coreWriter{} + w.i64(1) // MessageID + w.str("q") // Address + w.u8(0) // UserID len + w.u8(CoreTypeText) + w.u8(0) // Durable + w.i64(0) // Expiration + w.i64(0) // Timestamp + w.u8(0) // Priority + w.u8(0) // Large + w.u32(1) // one property + w.str("k") // key + w.u8(0xEE) // invalid value tag + w.blob(nil) // body + if _, err := DecodeCorePayload(w.b); err == nil { + t.Fatalf("want unknown-value-tag error, got nil") + } +} + +func TestFloat32BitsRoundTrip(t *testing.T) { + // guards the f32 encode/decode path independently of map iteration order. + orig := &CorePayload{Type: CoreTypeBytes, Properties: map[string]any{"f": float32(math.MaxFloat32)}} + got, _ := DecodeCorePayload(orig.Encode()) + if got.Properties["f"] != float32(math.MaxFloat32) { + t.Errorf("f32 = %v, want MaxFloat32", got.Properties["f"]) + } +} diff --git a/internal/journal/file.go b/internal/journal/file.go new file mode 100644 index 0000000..140f25f --- /dev/null +++ b/internal/journal/file.go @@ -0,0 +1,359 @@ +package journal + +import ( + "fmt" + "os" + "path/filepath" + "sort" +) + +// --- Journal record framing types (JournalImpl.java's outer per-record type +// byte, as opposed to the JournalRecordIds "user record type" table below). +// Verified against apache/activemq-artemis tag 2.42.0, JournalImpl.java +// lines 169-207; see format_notes.md section 2. Valid range is +// [EventRecord, RollbackRecord]; any other byte is padding/fill (e.g. the +// 'J' FILL_CHARACTER = 0x4A) and is skipped without comment. +const ( + EventRecord byte = 10 + AddRecord byte = 11 + UpdateRecord byte = 12 + AddRecordTx byte = 13 + UpdateRecordTx byte = 14 + DeleteRecordTx byte = 15 + DeleteRecord byte = 16 + PrepareRecord byte = 17 + CommitRecord byte = 18 + RollbackRecord byte = 19 +) + +// --- User record type ids (JournalRecordIds), read from the userRecordType +// byte carried by ADD/UPDATE-family journal records (see UserType on +// RawRecord). Verified against +// artemis-server/.../persistence/impl/journal/JournalRecordIds.java tag +// 2.42.0; see format_notes.md section 4. The full table is defined here +// (rather than per-consumer) so later decode stages (bindings.go, message.go, +// large.go) can share one source of truth instead of redeclaring literals. +const ( + GroupRecord byte = 20 + QueueBindingRecord byte = 21 + QueueStatusRecord byte = 22 + IDCounterRecord byte = 24 + AddressSettingRecord byte = 25 + SecuritySettingRecord byte = 26 + DivertRecord byte = 27 + BridgeRecord byte = 28 + AddLargeMessagePending byte = 29 + AddLargeMessage byte = 30 + AddMessage byte = 31 + AddRef byte = 32 + AcknowledgeRef byte = 33 + UpdateDeliveryCount byte = 34 + PageTransaction byte = 35 + SetScheduledDeliveryTime byte = 36 + DuplicateID byte = 37 + HeuristicCompletion byte = 38 + AcknowledgeCursor byte = 39 + PageCursorCounterValue byte = 40 + PageCursorCounterInc byte = 41 + PageCursorComplete byte = 42 + PageCursorPendingCounter byte = 43 + AddressBindingRecord byte = 44 + AddMessageProtocol byte = 45 + AddressStatusRecord byte = 46 + UserRecord byte = 47 + RoleRecord byte = 48 + AddMessageBody byte = 49 + KeyValuePairRecord byte = 50 + ConnectorRecord byte = 51 + AddressSettingRecordJSON byte = 52 + AckRetry byte = 53 +) + +// formatVersion is the only journal header format version this offline +// reader accepts. Real Artemis also tolerates the legacy value 1 +// (COMPATIBLE_VERSIONS), but the brief for this reader treats any mismatch +// as fatal: salvage targets 2.42 data directories, which are always written +// with format 2. format_notes.md section 1 / JournalImpl.FORMAT_VERSION. +const formatVersion = 2 + +// sizeHeader is the fixed journal file header length: int formatVersion + +// int userVersion + long fileID, in that write/read order (NOT the order +// implied by the misleading SIZE_HEADER summand comment in JournalImpl.java). +// format_notes.md section 1. +const sizeHeader = 16 + +// recordSizeOverhead is the fixed (non-variable) per-record byte count for +// journalVersion 2, i.e. JournalImpl.getRecordSize(recordType, 2): the +// leading type byte, 4-byte fileID echo, compactCount byte, any fixed-width +// fields (transactionID/recordID/userRecordType/numberOfRecords), and the +// trailing 4-byte checkSize -- everything except the variable-length body +// (and, for PREPARE_RECORD, the prepared-transaction extra data). +// +// Derived directly from JournalImpl.java's SIZE_* constants (tag 2.42.0) +// plus getRecordSize's "+1 for journalVersion>=2" adjustment. NOTE this +// corrects a transcription error in format_notes.md's DELETE_RECORD_TX entry +// (it states 21+1=22; the real Java constant SIZE_DELETE_RECORD_TX = +// BASIC_SIZE(9)+SIZE_LONG(8)+SIZE_LONG(8)+SIZE_INT(4) = 29, so the v2 +// overhead is 29+1=30). DELETE_RECORD_TX does not appear in the harvested +// fixture, so this correction is unverified against real bytes, but it is +// read straight from JournalImpl.java source and is internally consistent +// with every other entry in this table. +var recordSizeOverhead = map[byte]int{ + EventRecord: 23, + AddRecord: 23, + UpdateRecord: 23, + AddRecordTx: 31, + UpdateRecordTx: 31, + DeleteRecordTx: 30, + DeleteRecord: 18, + PrepareRecord: 26, + CommitRecord: 22, + RollbackRecord: 18, +} + +func isTransactionType(t byte) bool { + switch t { + case AddRecordTx, UpdateRecordTx, DeleteRecordTx, PrepareRecord, CommitRecord, RollbackRecord: + return true + default: + return false + } +} + +func isCompleteTransactionType(t byte) bool { + switch t { + case PrepareRecord, CommitRecord, RollbackRecord: + return true + default: + return false + } +} + +func isContainsBodyType(t byte) bool { + return t >= EventRecord && t <= DeleteRecordTx +} + +// RawRecord is one framed journal record, decoded to fields but with the +// user-record body left opaque. +type RawRecord struct { + Type byte // ADD_RECORD … ROLLBACK_RECORD (verified constants) + TxID int64 // tx-scoped records only + ID int64 // record id (message id / binding id); 0 for commit/rollback/prepare + UserType byte // user record type (JournalRecordIds); add/update family only + Body []byte // user body; add/update/delete-tx body; prepare extraData + NumberOfRecords int32 // commit/prepare only +} + +// FileDiag reports where and why scanning a file hit a corruption incident +// that required resyncing. +type FileDiag struct { + Path string + Offset int64 + Reason string + // Corrupt distinguishes structural journal/page/bindings damage (check-size + // mismatch, truncated record, broken page-entry framing, an undecodable + // bindings-record body) from benign diagnostic notes such as a + // fileID-mismatch "reuse leftover" from a normally-reused journal file + // (see parseRecord's "fileID mismatch" case below). salvage.go gates the + // CLI's exit code on Corrupt diags exactly like Skips (spec §1/§5); the + // zero value (false) is the safe default for a caller that has no reason + // to mark an entry corrupt. Set explicitly at every FileDiag construction + // site (this file, paging.go, bindings.go) rather than inferred later by + // matching against the assembled Reason prose. + Corrupt bool +} + +// ReadJournalDir orders *. files by fileID (from each header), +// streams every well-framed record to emit in replay order, and returns +// diagnostics for corruption incidents encountered along the way. A journal +// header format version other than the verified constant is an immediate +// error naming the file and both versions. +func ReadJournalDir(dir, prefix, ext string, emit func(RawRecord) error) ([]FileDiag, error) { + pattern := filepath.Join(dir, prefix+"*."+ext) + paths, err := filepath.Glob(pattern) + if err != nil { + return nil, fmt.Errorf("journal: glob %s: %w", pattern, err) + } + + type fileEntry struct { + path string + fileID int64 + data []byte + } + entries := make([]fileEntry, 0, len(paths)) + for _, p := range paths { + data, err := os.ReadFile(p) //nolint:gosec // salvage reads operator-supplied broker data dirs by design + if err != nil { + return nil, fmt.Errorf("journal: read %s: %w", p, err) + } + if len(data) < sizeHeader { + // Below SIZE_HEADER: damaged/empty, per format_notes.md section 1 + // (mirrors JournalImpl.readJournalFile's early -1 return). Not + // worth a diagnostic; a preallocated-but-never-written file looks + // exactly like this too. + continue + } + + hdr := newReader(data[:sizeHeader]) + gotVersion := hdr.i32() + _ = hdr.i32() // userVersion: echoed by the broker, not needed for framing + fileID := hdr.i64() + if hdr.err() != nil { + return nil, fmt.Errorf("journal: %s: reading header: %w", p, hdr.err()) + } + if gotVersion != formatVersion { + return nil, fmt.Errorf("journal: %s: unsupported journal format version %d (want %d)", p, gotVersion, formatVersion) + } + + entries = append(entries, fileEntry{path: p, fileID: fileID, data: data}) + } + + sort.Slice(entries, func(i, j int) bool { return entries[i].fileID < entries[j].fileID }) + + var diags []FileDiag + for _, e := range entries { + fileDiags, err := readJournalFile(e.path, e.data, e.fileID, emit) + diags = append(diags, fileDiags...) + if err != nil { + return diags, err + } + } + return diags, nil +} + +// readJournalFile scans one already-loaded journal file for well-framed +// records, streaming each to emit in on-disk order. +// +// It never aborts the file on a bad record; it mirrors +// JournalImpl.readJournalFile's "never stop the file" behavior +// (format_notes.md section 3): a record-type byte outside [EventRecord, +// RollbackRecord] is normal padding/fill and is skipped silently one byte at +// a time. A genuine corruption signal -- a fileID-echo mismatch, a field +// that would read past EOF, or a trailing check-size mismatch -- resyncs the +// same way, one byte at a time from the failed record's start, until a +// record parses cleanly again or the file ends. +// +// Diagnostics are coalesced per corruption incident rather than per resync +// byte: once a diagnostic has been recorded for the incident currently being +// recovered from, further failed attempts during that same recovery scan are +// silent, and the next successfully parsed record clears the state. Without +// this, a single corrupted record can spray one diagnostic per byte of its +// own body (any byte whose value happens to fall in the valid record-type +// range looks like a new candidate record start), which would drown out the +// single real incident it represents. +func readJournalFile(path string, data []byte, fileID int64, emit func(RawRecord) error) ([]FileDiag, error) { + var diags []FileDiag + fileIDEcho := int32(fileID) // low 32 bits; JournalFileImpl.getRecordID() truncates the long fileID to int + + pos := sizeHeader + resyncing := false + for pos < len(data) { + typeByte := data[pos] + if typeByte < EventRecord || typeByte > RollbackRecord { + pos++ + continue + } + + rec, consumed, reason, corrupt := parseRecord(data, pos, fileIDEcho, typeByte) + if reason != "" { + if !resyncing { + diags = append(diags, FileDiag{Path: path, Offset: int64(pos), Reason: reason, Corrupt: corrupt}) + resyncing = true + } + pos++ + continue + } + + resyncing = false + if err := emit(rec); err != nil { + return diags, fmt.Errorf("journal: %s: emit at offset %d: %w", path, pos, err) + } + pos += consumed + } + return diags, nil +} + +// parseRecord attempts to decode one record starting at pos. On success it +// returns the decoded record and the total number of bytes it occupies +// on-disk (including its trailing check-size int); the caller advances pos +// by that amount. On failure it returns a non-empty reason, a corrupt flag +// classifying that reason (true = structural damage, gates salvage's exit +// code; false = benign, report-only), and the caller resyncs by one byte. +// typeByte is data[pos], already verified to be in [EventRecord, +// RollbackRecord] by the caller. +func parseRecord(data []byte, pos int, fileIDEcho int32, typeByte byte) (RawRecord, int, string, bool) { + r := newReader(data[pos:]) + r.u8() // type byte, already known + + echo := r.i32() + if r.err() != nil { + return RawRecord{}, 0, "truncated record", true + } + if echo != fileIDEcho { + // Leftover bytes from a reused file, not corruption in the current + // generation -- but still worth a diagnostic for operator visibility + // (format_notes.md section 3's "CRITICAL" resync note; real Artemis + // stays silent here, our offline tool does not). Benign: never gates + // salvage's exit code. + return RawRecord{}, 0, "fileID mismatch", false + } + + r.u8() // compactCount (v2 journals only; format version is pinned to 2 above) + + var txID int64 + if isTransactionType(typeByte) { + txID = r.i64() + } + + var recordID int64 + if !isCompleteTransactionType(typeByte) { + recordID = r.i64() + } + + var variableSize int32 + var userType byte + var body []byte + if isContainsBodyType(typeByte) { + variableSize = r.i32() + if typeByte != DeleteRecordTx { + userType = r.u8() + } + body = r.bytes(int(variableSize)) + } + + var numberOfRecords int32 + var extraDataSize int32 + if typeByte == PrepareRecord || typeByte == CommitRecord { + numberOfRecords = r.i32() + if typeByte == PrepareRecord { + extraDataSize = r.i32() + body = r.bytes(int(extraDataSize)) + } + } + + checkSize := r.i32() + if r.err() != nil { + return RawRecord{}, 0, "truncated record", true + } + + overhead, ok := recordSizeOverhead[typeByte] + if !ok { + // Unreachable: typeByte is already verified to be in + // [EventRecord, RollbackRecord], and every value in that range has + // an entry in recordSizeOverhead. + return RawRecord{}, 0, "truncated record", true + } + total := overhead + int(variableSize) + int(extraDataSize) + if int(checkSize) != total { + return RawRecord{}, 0, "check-size mismatch", true + } + + return RawRecord{ + Type: typeByte, + TxID: txID, + ID: recordID, + UserType: userType, + Body: body, + NumberOfRecords: numberOfRecords, + }, total, "", false +} diff --git a/internal/journal/file_test.go b/internal/journal/file_test.go new file mode 100644 index 0000000..c227f49 --- /dev/null +++ b/internal/journal/file_test.go @@ -0,0 +1,237 @@ +package journal + +import ( + "encoding/binary" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReadJournalDirMessageJournal(t *testing.T) { + dir := fixtureDir(t) + journalDir := filepath.Join(dir, "journal") + + var records []RawRecord + diags, err := ReadJournalDir(journalDir, "activemq-data", "amq", func(r RawRecord) error { + records = append(records, r) + return nil + }) + if err != nil { + t.Fatalf("ReadJournalDir: %v", err) + } + if len(diags) != 0 { + t.Fatalf("want 0 diags over the clean fixture, got %d: %+v", len(diags), diags) + } + if len(records) == 0 { + t.Fatalf("want > 0 records, got 0") + } + + var sawAddMessageProtocol, sawAcknowledgeRef bool + for _, r := range records { + switch r.UserType { + case AddMessageProtocol: + sawAddMessageProtocol = true + case AcknowledgeRef: + sawAcknowledgeRef = true + } + } + if !sawAddMessageProtocol { + t.Errorf("want at least one record with UserType == AddMessageProtocol") + } + if !sawAcknowledgeRef { + t.Errorf("want at least one record with UserType == AcknowledgeRef") + } +} + +func TestReadJournalDirBindingsJournal(t *testing.T) { + dir := fixtureDir(t) + bindingsDir := filepath.Join(dir, "bindings") + + var records []RawRecord + diags, err := ReadJournalDir(bindingsDir, "activemq-bindings", "bindings", func(r RawRecord) error { + records = append(records, r) + return nil + }) + if err != nil { + t.Fatalf("ReadJournalDir: %v", err) + } + if len(diags) != 0 { + t.Fatalf("want 0 diags over the clean fixture, got %d: %+v", len(diags), diags) + } + + var queueBindings int + for _, r := range records { + if r.UserType == QueueBindingRecord { + queueBindings++ + } + } + if queueBindings == 0 { + t.Errorf("want >= 1 QueueBindingRecord, got 0") + } +} + +// recordSpan is a [start, end) byte range of one cleanly-parsed record, +// gathered by re-walking a file with the package's own framing logic. Tests +// use this instead of hand-decoded fixture hex offsets so corruption +// injection doesn't depend on manually tracking real broker output byte-for- +// byte. +type recordSpan struct{ start, end int } + +func scanCleanSpans(t *testing.T, data []byte) []recordSpan { + t.Helper() + hdr := newReader(data[:sizeHeader]) + hdr.i32() // formatVersion + hdr.i32() // userVersion + fileID := hdr.i64() + if hdr.err() != nil { + t.Fatalf("scanCleanSpans: reading header: %v", hdr.err()) + } + fileIDEcho := int32(fileID) + + var spans []recordSpan + pos := sizeHeader + for pos < len(data) { + typeByte := data[pos] + if typeByte < EventRecord || typeByte > RollbackRecord { + pos++ + continue + } + _, consumed, reason, _ := parseRecord(data, pos, fileIDEcho, typeByte) + if reason != "" { + t.Fatalf("scanCleanSpans: fixture file did not parse cleanly at offset %d: %s", pos, reason) + } + spans = append(spans, recordSpan{start: pos, end: pos + consumed}) + pos += consumed + } + return spans +} + +func copyFile(t *testing.T, dstDir, name string, data []byte) string { + t.Helper() + dst := filepath.Join(dstDir, name) + if err := os.WriteFile(dst, data, 0o644); err != nil { + t.Fatalf("write %s: %v", dst, err) + } + return dst +} + +func TestReadJournalDirCorruptRecordResyncs(t *testing.T) { + dir := fixtureDir(t) + srcPath := filepath.Join(dir, "journal", "activemq-data-1.amq") + data, err := os.ReadFile(srcPath) + if err != nil { + t.Fatalf("read fixture file: %v", err) + } + + spans := scanCleanSpans(t, data) + if len(spans) < 2 { + t.Fatalf("need >= 2 records in activemq-data-1.amq to exercise resync-with-survivors, got %d", len(spans)) + } + target := spans[1] // leave spans[0] as a pre-corruption survivor + + corrupted := append([]byte(nil), data...) + corruptOffset := target.end - 1 // last byte of the trailing check-size int + corrupted[corruptOffset] ^= 0xFF + + dstDir := t.TempDir() + dstPath := copyFile(t, dstDir, "activemq-data-1.amq", corrupted) + + var got []RawRecord + diags, err := ReadJournalDir(dstDir, "activemq-data", "amq", func(r RawRecord) error { + got = append(got, r) + return nil + }) + if err != nil { + t.Fatalf("ReadJournalDir: %v", err) + } + if len(diags) != 1 { + t.Fatalf("want exactly 1 diag for one corrupted record, got %d: %+v", len(diags), diags) + } + if diags[0].Path != dstPath { + t.Errorf("diag path = %q, want %q", diags[0].Path, dstPath) + } + if diags[0].Offset != int64(target.start) { + t.Errorf("diag offset = %d, want %d", diags[0].Offset, target.start) + } + if diags[0].Reason == "" { + t.Errorf("diag reason is empty") + } + if len(got) != len(spans)-1 { + t.Errorf("want %d surviving records (all but the corrupted one), got %d", len(spans)-1, len(got)) + } + if len(got) == 0 { + t.Fatalf("want records-before-corruption still emitted, got none") + } +} + +func TestReadJournalDirTruncatedRecordNoPanic(t *testing.T) { + dir := fixtureDir(t) + srcPath := filepath.Join(dir, "journal", "activemq-data-1.amq") + data, err := os.ReadFile(srcPath) + if err != nil { + t.Fatalf("read fixture file: %v", err) + } + + spans := scanCleanSpans(t, data) + if len(spans) < 2 { + t.Fatalf("need >= 2 records in activemq-data-1.amq to exercise truncation-with-survivors, got %d", len(spans)) + } + target := spans[1] + mid := target.start + (target.end-target.start)/2 + if mid <= target.start || mid >= target.end { + t.Fatalf("target record too small to truncate mid-record: %+v", target) + } + truncated := append([]byte(nil), data[:mid]...) + + dstDir := t.TempDir() + dstPath := copyFile(t, dstDir, "activemq-data-1.amq", truncated) + + var got []RawRecord + diags, err := ReadJournalDir(dstDir, "activemq-data", "amq", func(r RawRecord) error { + got = append(got, r) + return nil + }) + if err != nil { + t.Fatalf("ReadJournalDir: %v", err) + } + if len(diags) != 1 { + t.Fatalf("want exactly 1 diag for the truncated record, got %d: %+v", len(diags), diags) + } + if diags[0].Path != dstPath { + t.Errorf("diag path = %q, want %q", diags[0].Path, dstPath) + } + if diags[0].Offset != int64(target.start) { + t.Errorf("diag offset = %d, want %d", diags[0].Offset, target.start) + } + if len(got) == 0 { + t.Fatalf("want records-before-truncation still emitted, got none") + } +} + +func TestReadJournalDirWrongFormatVersionErrors(t *testing.T) { + dir := fixtureDir(t) + srcPath := filepath.Join(dir, "journal", "activemq-data-1.amq") + data, err := os.ReadFile(srcPath) + if err != nil { + t.Fatalf("read fixture file: %v", err) + } + + patched := append([]byte(nil), data...) + binary.BigEndian.PutUint32(patched[0:4], 99) // formatVersion := 99 + + dstDir := t.TempDir() + copyFile(t, dstDir, "activemq-data-1.amq", patched) + + _, err = ReadJournalDir(dstDir, "activemq-data", "amq", func(RawRecord) error { return nil }) + if err == nil { + t.Fatalf("want an error for a mismatched format version, got nil") + } + msg := err.Error() + if !strings.Contains(msg, "99") { + t.Errorf("error %q does not mention the file's version (99)", msg) + } + if !strings.Contains(msg, "2") { + t.Errorf("error %q does not mention the expected version (2)", msg) + } +} diff --git a/internal/journal/format_notes.md b/internal/journal/format_notes.md new file mode 100644 index 0000000..b0e2196 --- /dev/null +++ b/internal/journal/format_notes.md @@ -0,0 +1,548 @@ +# Artemis 2.42 on-disk format notes (verified against tag 2.42.0) + +Ground truth for the `salvage` offline reader. Every constant below was read from +`apache/activemq-artemis` at tag **2.42.0** (raw.githubusercontent.com). Byte +offsets cross-checked against the harvested fixture (`testdata/artemis-2.42-data.tar.gz`); +see "Fixture cross-check" at the bottom. **The fixture bytes are the final authority.** + +## Endianness (read this first) + +- All journal, bindings, and paging multi-byte **int/long/short** are **big-endian** + (Java `ByteBuffer` / Netty `ByteBuf` default order). This differs from the tool's + own `.artx` store (little-endian). +- `SimpleString` character data is stored as **little-endian UTF-16 pairs** (low byte, + then high byte) — see SimpleString section. The `int` length prefix in front of it + is still big-endian. +- `float` = big-endian int of `Float.floatToIntBits`; `double` = big-endian long of + `Double.doubleToLongBits`. +- **boolean true = 0xFF** (ActiveMQBuffer's `writeBoolean` writes -1, not 1; fixture: + the large-message durable byte is `ff`). Decode as nonzero = true. + +--- + +## 1. Journal file header (`SIZE_HEADER` = 16 bytes) + +Source: `JournalImpl.readFileHeader` / `writeHeader` / `SIZE_HEADER` +(`artemis-journal/.../core/journal/impl/JournalImpl.java`, lines 158, 3062-3130). + +`SIZE_HEADER = SIZE_LONG + SIZE_INT + SIZE_INT = 16`. Note the constant's *summands* +are ordered fileID+version+version, but the actual **write/read order** is: + +| off | size | field | value | +| --- | ---- | ----- | ----- | +| 0 | 4 | int formatVersion | **2** (`FORMAT_VERSION`; compatible: {1}) | +| 4 | 4 | int userVersion | broker user version (echoed, must match) | +| 8 | 8 | long fileID | file id (ordering id) | + +The stale comment at line 613 ("First long is the ordering timestamp") is wrong; the +loop just skips `SIZE_HEADER` bytes. There is **no** timestamp field. + +Files below `SIZE_HEADER` bytes are treated as damaged/empty (return -1, skip). +`MIN_FILE_SIZE = 1024`. Default journal files are named `activemq-data-N.amq`; +bindings journal files are `activemq-bindings-N.bindings` with the same header/framing. + +--- + +## 2. Record type bytes (the outer journal framing type) + +Source: `JournalImpl.java` lines 169-207. **`EVENT_RECORD=10` exists in 2.42.** + +| const | value | body? | tx? | complete-tx? | +| ----- | ----- | ----- | --- | ------------ | +| EVENT_RECORD | 10 | yes | no | no | +| ADD_RECORD | 11 | yes | no | no | +| UPDATE_RECORD | 12 | yes | no | no | +| ADD_RECORD_TX | 13 | yes | yes | no | +| UPDATE_RECORD_TX | 14 | yes | yes | no | +| DELETE_RECORD_TX | 15 | yes | yes | no | +| DELETE_RECORD | 16 | no | no | no | +| PREPARE_RECORD | 17 | no | no | yes | +| COMMIT_RECORD | 18 | no | no | yes | +| ROLLBACK_RECORD | 19 | no | no | yes | + +- **Valid record-type range = [10, 19]** (`recordType < EVENT_RECORD || recordType > ROLLBACK_RECORD` ⇒ skip). +- `isContainsBody(t)` = `t >= 10 && t <= 15` (EVENT/ADD/UPDATE/ADD_TX/UPDATE_TX/DELETE_TX). +- `isTransaction(t)` = ADD_TX/UPDATE_TX/DELETE_TX (13,14,15). +- `isCompleteTransaction(t)` = COMMIT/PREPARE/ROLLBACK (18,17,19). +- `FILL_CHARACTER = 'J'` (0x4A) — padding fill; 0x4A > 19 so it fails the range test and + is skipped like any other non-record byte. + +--- + +## 3. Per-record framing (`readJournalFile`, journalVersion >= 2) + +Source: `JournalImpl.readJournalFile` (lines 595-900) and `getRecordSize` (3040-3060). +Sizes: `BASIC_SIZE = SIZE_BYTE + SIZE_INT + SIZE_INT = 9` (type byte + fileID int + trailing +check-size int). In v2, `getRecordSize` adds +1 for the compactCount byte. + +### ADD_RECORD / UPDATE_RECORD / EVENT_RECORD (11/12/10) — carry user records + +| off | size | field | +| --- | ---- | ----- | +| 0 | 1 | recordType byte (10/11/12) | +| 1 | 4 | int fileID echo (see note) | +| 5 | 1 | byte compactCount (v>=2 only) | +| 6 | 8 | long recordID | +| 14 | 4 | int variableSize (= length of `record`) | +| 18 | 1 | byte userRecordType (JournalRecordIds, e.g. 45) | +| 19 | variableSize | record data (persister/codec encoded) | +| 19+variableSize | 4 | int checkSize = variableSize + recordSize | + +`recordSize` for ADD/UPDATE/EVENT (v2) = `SIZE_ADD_RECORD(22) + 1 = 23`. So +`checkSize = variableSize + 23`, and total on-disk record length = `23 + variableSize`. + +### *_RECORD_TX (13/14) — same but with transactionID before recordID + +Insert `[+8 long transactionID]` immediately after compactCount (off 6), shifting the +rest down 8. `recordSize` = `SIZE_ADD_RECORD_TX(30)+1 = 31`. + +### DELETE_RECORD_TX (15) + +type, fileID, compactCount, transactionID(8), recordID(8), int variableSize, record +(NO userRecordType byte — `recordType != DELETE_RECORD_TX` guards the userRecordType +read), checkSize. `recordSize = SIZE_DELETE_RECORD_TX(29)+1 = 30`. + +**Correction (Task 4, re-verified against JournalImpl.java tag 2.42.0 source +directly):** the value `21` above was a transcription error. The real constant is +`SIZE_DELETE_RECORD_TX = BASIC_SIZE(9) + SIZE_LONG(8) + SIZE_LONG(8) + SIZE_INT(4) = 29`, +so the v2 (compactCount-adjusted) recordSize is `29+1 = 30`, not `22`. This type does not +appear in the harvested fixture, so it is unverified against real bytes, but `29` is +read straight from source and is internally consistent with every other SIZE_* constant +in this section (each one is BASIC_SIZE plus the sum of its type-specific fixed fields). + +### DELETE_RECORD (16) + +type, fileID, compactCount, recordID(8), checkSize. No body. `recordSize = SIZE_DELETE_RECORD(17)+1 = 18`. + +### COMMIT_RECORD (18) / PREPARE_RECORD (17) / ROLLBACK_RECORD (19) + +No recordID (complete-transaction). transactionID(8) present. COMMIT/PREPARE also read +`int transactionCheckNumberOfRecords`; PREPARE additionally reads `int +preparedTransactionExtraDataSize` + that many extra bytes. ROLLBACK = type+fileID+ +compactCount+transactionID+checkSize. + +### fileID echo note + +The per-record fileID is read with `getInt()` (4 bytes) but the header fileID is a +`long`. The per-record echo is the **low 32 bits** of the file's long fileID +(`JournalFileImpl.getRecordID()` returns an int). + +### Resync / corruption rules (CRITICAL — corrects the plan's "stop file" wording) + +`readJournalFile` **never aborts the whole file** on a bad record. In every failure +case it repositions to `pos+1` (or `pos + SIZE_BYTE`) and keeps scanning to end-of-file: + +- recordType outside [10,19] ⇒ `continue` (position already at pos+1). Padding/holes/`'J'` fill land here. +- `isInvalidSize(...)` (a field read would run past the file) ⇒ `position = pos+1`, continue. +- **fileID mismatch** (`readFileId != file.getRecordID() && !reclaimed`) ⇒ leftover from a + reused file ⇒ `position = pos+1`, continue (skip this record, keep scanning). *Not* a stop. +- **checkSize mismatch** (trailing int != variableSize+recordSize+extra) ⇒ corruption ⇒ + `markAsDataFile`, `position = pos+1`, continue. *Not* a stop. + +So the offline reader should mirror this: resync byte-by-byte, never truncate the file +on a single bad record. (The harvested fixture is clean, so every record validates on +the first try; the resync path is exercised synthetically in later tasks.) + +--- + +## 4. User record type ids (`JournalRecordIds`) + +Source: `artemis-server/.../persistence/impl/journal/JournalRecordIds.java`. All values +match the plan's expectations. **No `EVENT_RECORD` here** (that lives in the framing type +table, section 2). Full table: + +| id | const | journal | used by salvage | +| -- | ----- | ------- | --------------- | +| 20 | GROUP_RECORD | msg | no | +| 21 | QUEUE_BINDING_RECORD | bindings | yes (queue id ↔ name/address) | +| 22 | QUEUE_STATUS_RECORD | bindings | no | +| 24 | ID_COUNTER_RECORD | bindings | no | +| 25 | ADDRESS_SETTING_RECORD | bindings | no | +| 26 | SECURITY_SETTING_RECORD | bindings | no | +| 27 | DIVERT_RECORD | bindings | no | +| 28 | BRIDGE_RECORD | bindings | no | +| 29 | ADD_LARGE_MESSAGE_PENDING | msg | no (deprecated) | +| 30 | ADD_LARGE_MESSAGE | msg | core large msgs only — **AMQP large msgs arrive as 45, see §5b** | +| 31 | ADD_MESSAGE | msg | yes (legacy core) | +| 32 | ADD_REF | msg | yes (msg↔queue placement) | +| 33 | ACKNOWLEDGE_REF | msg | yes (survivor elimination) | +| 34 | UPDATE_DELIVERY_COUNT | msg | optional | +| 35 | PAGE_TRANSACTION | msg | no (v1) | +| 36 | SET_SCHEDULED_DELIVERY_TIME | msg | yes (scheduled time via UPDATE) | +| 37 | DUPLICATE_ID | msg | no | +| 38 | HEURISTIC_COMPLETION | msg | no | +| 39 | ACKNOWLEDGE_CURSOR | msg | no (v1) | +| 40 | PAGE_CURSOR_COUNTER_VALUE | msg | no | +| 41 | PAGE_CURSOR_COUNTER_INC | msg | no | +| 42 | PAGE_CURSOR_COMPLETE | msg | no | +| 43 | PAGE_CURSOR_PENDING_COUNTER | msg | no | +| 44 | ADDRESS_BINDING_RECORD | bindings | no | +| 45 | ADD_MESSAGE_PROTOCOL | msg | **yes (the AMQP message add)** | +| 46 | ADDRESS_STATUS_RECORD | bindings | no | +| 47 | USER_RECORD | bindings | no | +| 48 | ROLE_RECORD | bindings | no | +| 49 | ADD_MESSAGE_BODY | msg | no (history) | +| 50 | KEY_VALUE_PAIR_RECORD | bindings | no | +| 51 | CONNECTOR_RECORD | bindings | no | +| 52 | ADDRESS_SETTING_RECORD_JSON | bindings | no | +| 53 | ACK_RETRY | msg | no | + +--- + +## 5. Persister ids (`PersisterIDs`) — CORRECTION vs plan + +Source: `artemis-server/.../core/persistence/PersisterIDs.java`. `MAX_PERSISTERS = 5`. +**The plan guessed V3=4; it is actually V3=5, and AMQPLargeMessage=4.** + +| id | persister | +| -- | --------- | +| 0 | CoreLargeMessagePersister | +| 1 | CoreMessagePersister | +| 2 | AMQPMessagePersister | +| 3 | AMQPMessagePersisterV2 | +| **4** | **AMQPLargeMessagePersister** | +| **5** | **AMQPMessagePersisterV3** | + +`MessagePersister.getPersister(id)` maps `persisters[id-1]`; `id==0 || id>5` ⇒ null. +The persister id is the **first byte** of the record data (`MessagePersister.encode` +writes `buffer.writeByte(getID())`). Artemis 2.42 writes AMQP standard messages with the +**V3** persister (id 5) and AMQP large messages with id 4. + +### 5a. AMQP standard message record data (ADD_MESSAGE_PROTOCOL, userType 45) + +Encode chain: `MessagePersister.encode` → `AMQPMessagePersister.encode` → +(V2) extra props → (V3) expiration. Source: +`AMQPMessagePersister.encode` (lines 57-63), `AMQPMessagePersisterV2.encode` (65-73), +`AMQPMessagePersisterV3.encode` (54-59), `AMQPStandardMessage.persist` (210-218). + +| off | size | field | present in | +| --- | ---- | ----- | ---------- | +| 0 | 1 | byte persisterID (2 / 3 / 5) | all | +| 1 | 8 | long messageID | all | +| 9 | 8 | long messageFormat (AMQP format) | all | +| 17 | var | nullableSimpleString address | all | +| . | 4 | int amqpSize (= internalPersistSize) | all | +| . | amqpSize | **raw AMQP-encoded message bytes** (Header…Footer) | all | +| . | 4 | int extraPropsSize | V2, V3 | +| . | extraPropsSize | TypedProperties extra props (if size != 0) | V2, V3 | +| . | 8 | long expiration | V3 only | + +`msg.AMQP` for Task 7 = the `amqpSize` bytes verbatim. The AMQP body is +**length-prefixed** by `int amqpSize`, so the extra-props/expiration tail is +unambiguous (do not treat "all remaining bytes" as the AMQP message). + +### 5b. AMQP large message record data (persisterID 4) — CORRECTION vs plan + +**FIXTURE FINDING: AMQP large messages are journaled under userType 45 +(ADD_MESSAGE_PROTOCOL) with persisterID 4 — NOT under userType 30 ADD_LARGE_MESSAGE.** +The fixture's 300 KiB message is an ADD_RECORD, userType 45, first data byte 4. +userType 30 is used for *core* large messages only. Dispatch on the persister id +byte, not the userType, to tell standard vs large AMQP messages apart. + +Source: `AMQPLargeMessagePersister.encode/decode` (lines 74-137), +`AMQPLargeMessage.saveEncoding/readSavedEncoding` (210-281). + +| off | size | field | +| --- | ---- | ----- | +| 0 | 1 | byte persisterID = 4 | +| 1 | 8 | long messageID | +| 9 | 1 | boolean durable | +| 10 | 8 | long messageFormat | +| 18 | var | nullableSimpleString address | +| . | 4 | int extraPropsSize (0 ⇒ none) | +| . | extraPropsSize | TypedProperties extra props | +| . | var | **saved encoding** (section positions/sizes interleaved with AMQP-encoded Header/MessageAnnotations/Properties/ApplicationProperties objects) | +| . | 8 | long expiration | +| . | 1 | boolean reencoded | + +The **large body itself is NOT in the journal**; it lives in +`data/large-messages/.msg`. The "saved encoding" block is a self-delimiting +sequence of `int` position/size fields interleaved with AMQP object encodings (9 ints: +headerPosition, encodedHeaderSize, [Header], deliveryAnnotationsPosition, +encodedDeliveryAnnotationsSize, messageAnnotationsPosition, [MessageAnnotations], +propertiesPosition, [Properties], applicationPropertiesPosition, remainingBodyPosition, +[ApplicationProperties]). Positions may be -1 (section absent); an absent AMQP object +encodes as the single AMQP null byte `0x40`. + +**Fixture-verified join rule for Task 8**: `data/large-messages/.msg` holds +the **complete AMQP-encoded message** (Header + Properties + … + Data section with the +full body) — fixture `64.msg` is 307255 bytes = 55 bytes of sections + 307200 body, and +it starts with `00 53 70` (Header descriptor), not raw body bytes. So +`msg.AMQP = the .msg file bytes verbatim`; the journal record's saved-encoding block is +only a section index / header cross-check. Fixture extra props observed: +`_AMQ_AD` (STRING) = the original address name. + +### 5c. CoreMessage (persister id 1, userType 45) — DECODED + +**Verified byte-for-byte against a real 2.42.0 record** harvested with `artemis +producer --protocol CORE` (committed as `testdata/core-record-2.42.bin`, a BYTES +message: messageID 28, address `salvage.core`, 120-byte body). Modern 2.42 writes core +messages as **ADD_MESSAGE_PROTOCOL (userType 45) with persister id 1**, not userType 31. + +`CoreMessagePersister.encode` writes a `messageID` + `address` prefix, then delegates to +`CoreMessage.persist` (`writeInt(buffer.writerIndex())` + the buffer). The buffer holds +`endOfBodyPosition`, the body, then `encodeHeadersAndProperties` at the end: + +| off | size | field | +| --- | ---- | ----- | +| 0 | 1 | byte persisterID = 1 | +| 1 | 8 | long messageID (prefix; redundant with headers) | +| . | var | nullableSimpleString address (prefix; redundant with headers) | +| . | 4 | int bufferSize (`message.persist` length prefix) | +| . | 4 | int endOfBodyPosition (CoreMessage buffer[0..4)) | +| . | endOfBodyPosition − 13 | **message body bytes** | +| . | var | encodeHeadersAndProperties (below) | + +`encodeHeadersAndProperties`: `long messageID`, `nullableSimpleString address`, `byte +userID-null-flag` (+16-byte UUID when NOT_NULL), `byte type`, `boolean durable`, `long +expiration`, `long timestamp`, `byte priority`, `TypedProperties`. + +Constants (verified): **`BUFFER_HEADER_SPACE = PacketImpl.PACKET_HEADERS_SIZE = 13`** +(SIZE_INT + SIZE_BYTE + SIZE_LONG), `BODY_OFFSET = 4`. Body = `buffer[BODY_OFFSET : +endOfBodyPosition − BUFFER_HEADER_SPACE + BODY_OFFSET]` = `buffer[4 : +endOfBodyPosition − 9]`; headers follow. Core `type` byte: 0 DEFAULT, 2 OBJECT, 3 TEXT, +4 BYTES, 5 MAP, 6 STREAM. Decoded by `decodeCoreStandardBody` (message.go). + +**Core large message (userType 30):** the record body is `encodeHeadersAndProperties` +directly (no persister-id / endOfBodyPosition / bufferSize prefix); the message body +lives in `data/large-messages/.msg` as raw bytes (verified: `48.msg` = 307200 bytes +for a 300 KiB BYTES message). Decoded by `decodeCoreHeaders`, body joined by +`AttachLargeBodies`. + +**Paged core:** a page entry with `largeMessageType = NONE` and persister id 1 is the +same `CoreMessagePersister` payload, decoded by the same path; `largeMessageType = +CORE/OLD_CORE` is a core-large header whose body is outside the page file (still skipped). + +**Legacy ADD_MESSAGE (userType 31):** pre-persister-id core add, not produced by 2.42 +and unverified — still reported as a skip rather than best-effort decoded. + +--- + +## 6. SimpleString byte layout + +Source: `SimpleString` `getData`/`writeSimpleString`/`writeNullableSimpleString` +(lines 143-151, 263-276) and `readSimpleString` (242-261). + +**nullableSimpleString**: + +| off | size | field | +| --- | ---- | ----- | +| 0 | 1 | byte flag: 0 = NULL (stop), 1 = NOT_NULL | +| 1 | 4 | int byteLength (= 2 × charCount) | +| 5 | byteLength | UTF-16 chars, **little-endian pairs** (low byte, high byte) | + +**simpleString** (non-nullable, e.g. TypedProperties keys / STRING values): the same +minus the leading flag byte (int byteLength + LE char pairs). + +Char decode: `char = (data[j] & 0xFF) | ((data[j+1] << 8) & 0xFF00)` — index j is the low +byte. ASCII strings therefore appear as ` 0x00 0x00 …` in a hexdump. + +--- + +## 7. TypedProperties layout + DataConstants type ids + +Source: `TypedProperties.encode` (661-692), value writers (865-1193), +`DataConstants` (all values). + +**DataConstants type ids** (all verified): + +| id | const | +| -- | ----- | +| 0 | NULL | +| 1 | NOT_NULL | +| 2 | BOOLEAN | +| 3 | BYTE | +| 4 | BYTES | +| 5 | SHORT | +| 6 | INT | +| 7 | LONG | +| 8 | FLOAT | +| 9 | DOUBLE | +| 10 | STRING | +| 11 | CHAR | + +**TypedProperties.encode**: + +| off | size | field | +| --- | ---- | ----- | +| 0 | 1 | byte: NULL(0) ⇒ empty, stop; NOT_NULL(1) ⇒ continue | +| 1 | 4 | int propertyCount | +| . | .. | repeated `propertyCount` times: [int keyByteLen][key bytes = SimpleString data, LE pairs, NO flag][value] | + +**value** = 1 type byte + payload: + +| type | byte | payload | +| ---- | ---- | ------- | +| NULL | 0 | (none) | +| BOOLEAN | 2 | 1 byte | +| BYTE | 3 | 1 byte | +| BYTES | 4 | int length + bytes | +| SHORT | 5 | 2 bytes (BE) | +| INT | 6 | 4 bytes (BE) | +| LONG | 7 | 8 bytes (BE) | +| FLOAT | 8 | 4 bytes (BE int of floatToIntBits) | +| DOUBLE | 9 | 8 bytes (BE long of doubleToLongBits) | +| STRING | 10 | writeSimpleString (int byteLen + LE pairs, NO flag) | +| CHAR | 11 | 2 bytes (BE short) | + +Note: keys are raw `SimpleString` (int length + data), **not** nullableSimpleString — no +leading flag byte. + +--- + +## 8. Page file + PagedMessage layout + +Sources: `PageReadWriter` (`START_BYTE`/`END_BYTE`/`SIZE_RECORD`, lines 45-97), +`PagedMessageImpl.decode/encode` (200-265). Page files: `data/paging//.page`. + +**Per page entry** (`SIZE_RECORD = 1+4+1 = 6` overhead): + +| off | size | field | +| --- | ---- | ----- | +| 0 | 1 | START_BYTE = `'{'` = 0x7B | +| 1 | 4 | int messageEncodedSize (= PagedMessage encode size) | +| 5 | messageEncodedSize | PagedMessage bytes (below) | +| 5+size | 1 | END_BYTE = `'}'` = 0x7D | + +Reader validates START_BYTE, reads size, checks `pos+6+size <= fileSize`, and verifies the +trailing byte is END_BYTE. A bad start/end byte marks the file suspect (partial trailing +write tolerated). + +**PagedMessage** (`PagedMessageImpl`): + +| off | size | field | +| --- | ---- | ----- | +| 0 | 8 | long transactionID (**fixture: -1 = non-transactional**; treat <= 0 as no tx) | +| 8 | 1 | byte largeMessageType: 0=NONE, 1=CORE, -1=OLD_CORE, 2=NOT_CORE | +| . | .. | if type ∈ {CORE, OLD_CORE}: int coreLargeHeaderSize + that many core-large-persister header bytes. Otherwise: `MessagePersister.decode` = [byte persisterID][persister payload from §5] (persisterID 2/3/5 for normal AMQP, 4 for AMQP large) | +| . | 4 | int queueIDsCount | +| . | 8×count | long queueIDs[] | + +For our fixture (`salvage.paged`, plain 1 KiB AMQP messages): transactionID=-1, +largeMessageType=0 (NONE), then `[persisterID=5][V3 AMQP standard message]`, then queueIDs. + +**Paging dir layout (fixture-verified)**: `data/paging//` — one UUID-named dir per +paged address, containing `000000001.page`, `000000002.page`, … and **`address.txt`** +whose single line is the address name (fixture: `salvage.paged`). Use address.txt to map +the dir to its address. + +**Paging is a spillover, not a mirror (fixture-verified)**: messages sent *before* the +address crossed maxSizeBytes live in the journal like normal messages (fixture: 44 of +the 500 salvage.paged messages are journal ADD_MESSAGE_PROTOCOL records; the other 456 +are page entries). Replay must union journal survivors + page entries per queue. + +--- + +## 9. Reference read path (`XmlDataExporter` / `RecoverMessages`) — replay decisions to mirror + +Source: `XmlDataExporter.processMessageJournal` (185-296), `removeAcked` (288-...). + +1. `messageJournal.load(records, preparedTransactions, failureCb, false)` — the Journal's + own load applies **transaction semantics**: DELETE_RECORD removes the add; COMMIT + applies the tx's records; ROLLBACK and **unterminated** transactions are discarded; + PREPARE-only tx go into `preparedTransactions` (in-doubt). XmlDataExporter **discards + `preparedTransactions`** (in-doubt messages are NOT exported). Mirror this: only + committed / non-transactional adds survive. +2. Build `messages[messageID]` from ADD_MESSAGE (31), ADD_MESSAGE_PROTOCOL (45), + ADD_LARGE_MESSAGE (30). +3. Build `messageRefs[messageID][queueID]` from ADD_REF (32) records (RefEncoding = + long queueID). This is how a message maps to its queue(s). Fixture: ADD_REF arrives + as **UPDATE_RECORD** frames (type 12) with recordID = the messageID; treat adds and + updates uniformly when collecting user records (as `RecordInfo` does). +4. Collect ACKNOWLEDGE_REF (33) records; in `removeAcked`, for each ack remove + `messageRefs[id][queueID]`; if a message has **no remaining refs**, drop the message + entirely. ⇒ **acked messages are eliminated** (our `salvage.acked` set). +5. SET_SCHEDULED_DELIVERY_TIME (36) arrives as an UPDATE record carrying + ScheduledDeliveryEncoding = `long queueID` + `long scheduledDeliveryTime`. + +### Supporting codec field orders (verified) + +- **RefEncoding / QueueEncoding** (`QueueEncoding.decode`): `long queueID` only. +- **ScheduledDeliveryEncoding.decode**: `long queueID` then `long scheduledDeliveryTime`. +- **PersistentQueueBindingEncoding.decode** (queue binding, userType 21): `SimpleString + queueName`, `SimpleString address`, `nullableSimpleString filterString`, + `nullableSimpleString metadata (user)`, `boolean autoCreated`, then a versioned tail + of flags (maxConsumers int, purgeOnNoConsumers bool, routingType byte, …). Task 6 + finalizes the full tail; the queueName/address/filter/autoCreated prefix is stable and + is all salvage needs to map queueID → name/address. + +--- + +## 10. Page-cursor journal records (Task 9 addition -- not exercised by the fixture) + +The harvested fixture's message journal contains **zero** ACKNOWLEDGE_CURSOR (39) or +PAGE_CURSOR_COMPLETE (42) records (section 3's census: the only page-cursor-family +record present is one `ADD_TX/40` PAGE_CURSOR_COUNTER_VALUE, which is page-count +bookkeeping, not a per-message/per-page ack -- consistent with "nothing consumed" in +the fixture). Task 7 explicitly deferred these record families (see the design note in +`message.go`'s `DecodeMessages` doc comment). The layout below was **not** available in +this document and was pulled from `apache/activemq-artemis` tag **2.42.0** source +(not fixture-verified; recorded here per the "derive from Java sources, cite, then +code" rule) rather than guessed. + +**Both record types share one encoding class, `CursorAckRecordEncoding`** (source: +`artemis-server/.../persistence/impl/journal/codec/CursorAckRecordEncoding.java`, +`getEncodeSize`/`encode`/`decode`): + +| off | size | field | +| --- | ---- | ----- | +| 0 | 8 | long queueID | +| 8 | 8 | long pageNr (from `PagePosition.getPageNr()`) | +| 16 | 4 | int messageNr (from `PagePosition.getMessageNr()`) | + +Total 20 bytes, no variable-length parts. + +**Write path** (source: `AbstractJournalStorageManager.java`, tag 2.42.0): + +- `storeCursorAcknowledge(queueID, position)`: `messageJournal.appendAddRecord(freshID, + ACKNOWLEDGE_CURSOR, new CursorAckRecordEncoding(queueID, position), ...)` -- **ADD** + frame (type 11 non-tx), record ID is a **freshly generated id-generator value**, not + the paged message's own id and not the queueID. `storeCursorAcknowledgeTransactional` + is the same but via `appendAddRecordTransactional` (ADD_RECORD_TX, type 13). +- `storePageCompleteTransactional(txID, queueID, position)`: same encoding class, same + fresh-ID ADD_RECORD_TX pattern, userRecordType PAGE_CURSOR_COMPLETE instead. +- Practical consequence for replay: these are ordinary top-level Survivors (own record + ID, no relation to the message/page they describe) that ride through + `Replayer`/`ReadJournalDir` exactly like message records; `DecodeMessages` already + ignores them via its default case (design note above). `BuildCursorState` + (`paging.go`) re-scans the same `[]Survivor` for `UserType` 39/42 and decodes each + Body with the 20-byte layout above. + +**`messageNr` is not stored in the page file** (cross-check against section 8's +`PagedMessage` field table: no such field exists on disk). Source: `Page.java`'s +`addMessage(PagedMessage message)`, called once per entry as `PageReadWriter` decodes a +`.page` file in order: `message.setMessageNumber(messages.size())` -- i.e. **messageNr +is the entry's 0-based ordinal position within its page file, assigned at read time**, +matching the first entry decoded from a page to messageNr 0, the second to 1, etc. This +is the same value later wrapped in a `PagePositionImpl(pageNr, messageNr)` when the +broker acks/completes a cursor position, so `ReadPaging` reconstructing the same +ordinal while walking a `.page` file's entries in on-disk order reproduces the position +cursor acks reference. + +--- + +## Fixture cross-check results + +Fixture: `testdata/artemis-2.42-data.tar.gz` from `make fixtures` (2.42.0-alpine, +clean SIGTERM stop). A byte-walker implementing exactly the framing above parsed +**every record in both journals with 0 check-size failures** (one expected resync in +the bindings journal on a compaction leftover, recovered per §3's rules). + +- Header `activemq-data-1.amq`: `00 00 00 02 | 00 00 00 00 | 00 00 00 00 00 00 00 01` + = formatVersion 2, userVersion 0, fileID 1. ✓ §1. (`activemq-data-2.amq`: fileID 2, + all-zero body = preallocated, parses to 0 records. ✓ padding rule.) +- First record at 0x10: `0b | 00 00 00 01 | 00 | …recordID 38… | 00 00 00 ea | 2d | …` + = ADD_RECORD, fileID echo 1, compactCount 0, variableSize 234, userType 45, + persister byte 5 (V3), messageID echo 38; trailing checkSize 234+23. ✓ §3, §5a. +- Message journal census: 59× ADD/45 (58× persister 5, 1× persister 4 = the large msg), + 59× UPDATE/32 (ADD_REF arrives as UPDATE_RECORD frames), 1× UPDATE/36 (scheduled), + 3× UPDATE_TX/33 + COMMITs (acks are transactional), 3× DELETE_RECORD (the fully-acked + messages), 1× ADD_TX/40. ✓ §4, §9 (adds+deletes+acks all present for replay tests). +- Large message: journal record userType 45 / persister 4, durable byte `ff`, + address `salvage.large`, expiration 0, reencoded 0; `64.msg` = 307255 B complete AMQP + message (starts `00 53 70`, Data section `00 53 75 b0` + int 307200). ✓ §5b. +- Paging: 36 page files under `paging//` + `address.txt` = `salvage.paged`; + 456 entries total, every entry `7b | int size | … | 7d`, transactionID -1, + largeMessageType 0, persister 5; 456 + 44 journal-resident = 500 = manifest count; + a paged entry's AMQP Data payload sha256 matches its manifest entry. ✓ §8. +- Manifest: plain 5, props 5, scheduled 1, large 1, paged 500, acked 0. ✓ brief schema. diff --git a/internal/journal/harvest_integration_test.go b/internal/journal/harvest_integration_test.go new file mode 100644 index 0000000..8f1058e --- /dev/null +++ b/internal/journal/harvest_integration_test.go @@ -0,0 +1,482 @@ +package journal_test + +// Harvests a real Artemis 2.42 data dir into testdata/. Run via `make fixtures`. +// +// This is a fixture *generator* that happens to run under `go test` so it can +// reuse the repo's testcontainers plumbing. It is gated behind +// ARTEMISCTL_HARVEST=1 so `make test` never runs it. It uses a DEDICATED +// container (not brokertest.Shared): this flow stops the broker, which would +// break the shared container's Reuse contract. +// +// Output: +// - testdata/artemis-2.42-data.tar.gz : the broker's complete data/ dir +// (bindings/ journal/ large-messages/ paging/), tar paths rooted at "data/" +// - testdata/manifest.json : every produced message (sha256/len/props) + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Azure/go-amqp" + "github.com/docker/docker/api/types/container" + tc "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + + "github.com/martikan/artemisctl/internal/broker" + "github.com/martikan/artemisctl/internal/brokertest" +) + +// instanceDataDir is where the apache/activemq-artemis image keeps the broker +// instance's data directory; verified by exec'ing `ls` in the running +// container (the harvest fails loudly if the layout differs). +const instanceDataDir = "/var/lib/artemis-instance/data" + +// manifestEntry describes one produced message, schema per the task brief. +type manifestEntry struct { + BodySha256 string `json:"bodySha256"` + BodyLen int `json:"bodyLen"` + Props map[string]any `json:"props,omitempty"` + ScheduledAtMs int64 `json:"scheduledAtMs,omitempty"` +} + +type manifest struct { + Queues map[string][]manifestEntry `json:"queues"` +} + +// scheduledAtMs is 2100-01-01T00:00:00Z in unix millis — far enough in the +// future that the scheduled message never becomes deliverable during harvest. +const scheduledAtMs = int64(4102444800000) + +// fixtureBody builds the deterministic body for message i of a queue: +// fmt.Sprintf("salvage-fixture-%s-%04d|", queue, i) repeated (and truncated) +// to exactly n bytes. +func fixtureBody(queue string, i, n int) []byte { + unit := fmt.Sprintf("salvage-fixture-%s-%04d|", queue, i) + b := make([]byte, 0, n+len(unit)) + for len(b) < n { + b = append(b, unit...) + } + return b[:n] +} + +func sha256Hex(b []byte) string { + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} + +func TestHarvestFixture(t *testing.T) { + if os.Getenv("ARTEMISCTL_HARVEST") != "1" { + t.Skip("fixture harvester; run via make fixtures") + } + ctx := context.Background() + + // 1. Dedicated broker with a named volume over its data dir, so the data + // survives container removal and can be copied out by a helper container. + vol := fmt.Sprintf("artemisctl-harvest-%d", time.Now().Unix()) + req := tc.ContainerRequest{ + Image: "apache/activemq-artemis:2.42.0-alpine", + ExposedPorts: []string{"61616/tcp"}, + Env: map[string]string{ + "ARTEMIS_USER": "artemis", + "ARTEMIS_PASSWORD": "artemis", + // --nio: AIO fails under rootless container runtimes (see + // brokertest). --relax-jolokia keeps management reachable. + "EXTRA_ARGS": "--nio --relax-jolokia", + }, + HostConfigModifier: func(hc *container.HostConfig) { + hc.Binds = append(hc.Binds, vol+":"+instanceDataDir) + }, + WaitingFor: wait.ForListeningPort("61616/tcp").WithStartupTimeout(120 * time.Second), + } + ctr, err := tc.GenericContainer(ctx, tc.GenericContainerRequest{ContainerRequest: req, Started: true}) + if err != nil { + t.Fatalf("start artemis: %v", err) + } + defer func() { + _ = ctr.Terminate(context.Background()) + // The named volume is not managed by testcontainers; remove it by hand. + _ = exec.Command("docker", "volume", "rm", "-f", vol).Run() + }() + + // Verify the image's data-dir layout before trusting it: the journal must + // be created under the mounted path or the harvest would capture nothing. + requireDataDirLayout(t, ctx, ctr) + + host, err := ctr.Host(ctx) + if err != nil { + t.Fatalf("host: %v", err) + } + port, err := ctr.MappedPort(ctx, "61616/tcp") + if err != nil { + t.Fatalf("port: %v", err) + } + props := broker.ConnectionProps{URL: host + ":" + port.Port(), Username: "artemis", Password: "artemis"} + + connectCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + c, err := broker.Connect(connectCtx, props) + cancel() + if err != nil { + t.Fatalf("connect: %v", err) + } + + // 2. Force paging on salvage.paged BEFORE producing to it: 500 x 1 KiB + // >> 64 KiB maxSize with the PAGE policy => multiple 16 KiB page files. + pagedSettings, err := withPagingPolicy(brokertest.PermissiveWildcardSettings, 65536, 16384) + if err != nil { + t.Fatalf("build paged settings: %v", err) + } + opCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + err = c.ApplyAddressSettings(opCtx, "salvage.paged", pagedSettings) + cancel() + if err != nil { + t.Fatalf("apply paging settings: %v", err) + } + + // 3. Produce the known message set. Every body is deterministic; the + // manifest records sha256 + len (+ props / scheduled time). + man := manifest{Queues: map[string][]manifestEntry{}} + sendCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + // salvage.plain: 5 durable text messages, 64 B. + man.Queues["salvage.plain"] = sendBatch(t, sendCtx, c, "salvage.plain", 5, 64, nil) + + // salvage.props: 5 durable messages with application properties and priority 7. + man.Queues["salvage.props"] = sendBatch(t, sendCtx, c, "salvage.props", 5, 64, func(m *amqp.Message, e *manifestEntry) { + m.ApplicationProperties = map[string]any{"region": "eu", "attempt": int32(1)} + m.Header.Priority = 7 + e.Props = map[string]any{"region": "eu", "attempt": 1} + }) + + // salvage.scheduled: 1 durable message scheduled for 2100-01-01. + man.Queues["salvage.scheduled"] = sendBatch(t, sendCtx, c, "salvage.scheduled", 1, 64, func(m *amqp.Message, e *manifestEntry) { + m.Annotations = amqp.Annotations{"x-opt-delivery-time": scheduledAtMs} + e.ScheduledAtMs = scheduledAtMs + }) + + // salvage.large: 1 durable message, 300 KiB — over the 100 KiB + // amqpMinLargeMessageSize default, so stored as a large message. + man.Queues["salvage.large"] = sendBatch(t, sendCtx, c, "salvage.large", 1, 300*1024, nil) + + // salvage.acked: 3 messages, then receive and accept all 3 — leaves + // ADD_REF + ACKNOWLEDGE_REF material so replay can eliminate them. + sendBatch(t, sendCtx, c, "salvage.acked", 3, 64, nil) + receiveAndAccept(t, sendCtx, c, "salvage.acked", 3) + man.Queues["salvage.acked"] = []manifestEntry{} + + // salvage.paged: 500 durable 1 KiB messages into the paging-forced address. + man.Queues["salvage.paged"] = sendBatch(t, sendCtx, c, "salvage.paged", 500, 1024, nil) + + closeCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + err = c.Close(closeCtx) + cancel() + if err != nil { + t.Fatalf("close client: %v", err) + } + + // 4. Verify paging actually happened before freezing the fixture. + requireNonEmptyPaging(t, ctx, ctr) + + // 5. Clean shutdown with a generous timeout so Artemis closes the journal + // cleanly (SIGTERM => broker Stop). + stopTimeout := 60 * time.Second + if err := ctr.Stop(ctx, &stopTimeout); err != nil { + t.Fatalf("stop broker: %v", err) + } + + // 6. Copy the volume out through a busybox helper (the volume itself is + // not directly readable from the host under rootless runtimes). + out := copyVolumeOut(t, ctx, vol) + + // 7. Tar+gzip => testdata/artemis-2.42-data.tar.gz, rooted at "data/". + testdata := filepath.Join(mustModuleDir(t), "internal", "journal", "testdata") + if err := os.MkdirAll(testdata, 0o755); err != nil { + t.Fatalf("mkdir testdata: %v", err) + } + tarball := filepath.Join(testdata, "artemis-2.42-data.tar.gz") + if err := tarGzDir(out, "data", tarball); err != nil { + t.Fatalf("pack tarball: %v", err) + } + fi, err := os.Stat(tarball) + if err != nil { + t.Fatalf("stat tarball: %v", err) + } + t.Logf("tarball: %s (%d bytes)", tarball, fi.Size()) + if fi.Size() > 5*1024*1024 { + t.Fatalf("fixture tarball is %d bytes (> 5 MB): too big to commit, investigate", fi.Size()) + } + + // 8. Write the manifest. + mb, err := json.MarshalIndent(man, "", " ") + if err != nil { + t.Fatalf("marshal manifest: %v", err) + } + if err := os.WriteFile(filepath.Join(testdata, "manifest.json"), append(mb, '\n'), 0o644); err != nil { + t.Fatalf("write manifest: %v", err) + } +} + +// sendBatch sends n durable messages of bodyLen bytes to queue and returns +// their manifest entries. customize (optional) mutates each message and its +// manifest entry before sending. +func sendBatch(t *testing.T, ctx context.Context, c *broker.Client, queue string, n, bodyLen int, customize func(*amqp.Message, *manifestEntry)) []manifestEntry { + t.Helper() + // TargetCapabilities "queue" is REQUIRED: without it Artemis routes the + // send as multicast and the message never lands in the anycast queue. + sender, err := c.Session().NewSender(ctx, queue, &amqp.SenderOptions{TargetCapabilities: []string{"queue"}}) + if err != nil { + t.Fatalf("open sender %s: %v", queue, err) + } + defer sender.Close(context.Background()) + + entries := make([]manifestEntry, 0, n) + for i := 0; i < n; i++ { + body := fixtureBody(queue, i, bodyLen) + msg := amqp.NewMessage(body) + msg.Header = &amqp.MessageHeader{Durable: true} + msg.Properties = &amqp.MessageProperties{MessageID: fmt.Sprintf("%s-%04d", queue, i)} + e := manifestEntry{BodySha256: sha256Hex(body), BodyLen: len(body)} + if customize != nil { + customize(msg, &e) + } + if err := sender.Send(ctx, msg, nil); err != nil { + t.Fatalf("send %s[%d]: %v", queue, i, err) + } + entries = append(entries, e) + } + return entries +} + +// receiveAndAccept receives and accepts n messages from queue. +func receiveAndAccept(t *testing.T, ctx context.Context, c *broker.Client, queue string, n int) { + t.Helper() + recv, err := c.Session().NewReceiver(ctx, queue, &amqp.ReceiverOptions{ + SourceCapabilities: []string{"queue"}, + }) + if err != nil { + t.Fatalf("open receiver %s: %v", queue, err) + } + defer recv.Close(context.Background()) + for i := 0; i < n; i++ { + msg, err := recv.Receive(ctx, nil) + if err != nil { + t.Fatalf("receive %s[%d]: %v", queue, i, err) + } + if err := recv.AcceptMessage(ctx, msg); err != nil { + t.Fatalf("accept %s[%d]: %v", queue, i, err) + } + } +} + +// withPagingPolicy returns base (a settings JSON object) with the address-full +// policy forced to PAGE and the given size thresholds, leaving all other +// fields (DLA, expiry, auto-create, ...) intact. +func withPagingPolicy(base string, maxSizeBytes, pageSizeBytes int) (string, error) { + var m map[string]json.RawMessage + if err := json.Unmarshal([]byte(base), &m); err != nil { + return "", fmt.Errorf("parse base settings: %w", err) + } + m["addressFullMessagePolicy"] = json.RawMessage(`"PAGE"`) + m["maxSizeBytes"] = json.RawMessage(fmt.Sprintf("%d", maxSizeBytes)) + m["pageSizeBytes"] = json.RawMessage(fmt.Sprintf("%d", pageSizeBytes)) + out, err := json.Marshal(m) + if err != nil { + return "", fmt.Errorf("marshal paged settings: %w", err) + } + return string(out), nil +} + +// execOut runs cmd in the container and returns its combined output. +func execOut(t *testing.T, ctx context.Context, ctr tc.Container, cmd []string) (int, string) { + t.Helper() + code, rd, err := ctr.Exec(ctx, cmd) + if err != nil { + t.Fatalf("exec %v: %v", cmd, err) + } + b, err := io.ReadAll(rd) + if err != nil { + t.Fatalf("exec %v read: %v", cmd, err) + } + return code, string(b) +} + +// requireDataDirLayout fails the harvest unless the running broker keeps its +// journal under instanceDataDir (i.e. under our named volume). +func requireDataDirLayout(t *testing.T, ctx context.Context, ctr tc.Container) { + t.Helper() + code, out := execOut(t, ctx, ctr, []string{"ls", instanceDataDir}) + if code != 0 { + _, alt := execOut(t, ctx, ctr, []string{"ls", "/var/lib/artemis-instance"}) + t.Fatalf("data dir %s not found (exit %d); instance layout:\n%s", instanceDataDir, code, alt) + } + for _, want := range []string{"journal", "bindings", "large-messages", "paging"} { + if !strings.Contains(out, want) { + t.Fatalf("data dir %s missing %q; contents:\n%s", instanceDataDir, want, out) + } + } +} + +// requireNonEmptyPaging fails the harvest unless the paging dir contains a +// non-empty address dir with at least 2 page files. +func requireNonEmptyPaging(t *testing.T, ctx context.Context, ctr tc.Container) { + t.Helper() + code, out := execOut(t, ctx, ctr, []string{"sh", "-c", "ls " + instanceDataDir + "/paging/*/ 2>/dev/null"}) + if code != 0 || strings.TrimSpace(out) == "" { + _, top := execOut(t, ctx, ctr, []string{"ls", "-la", instanceDataDir + "/paging"}) + t.Fatalf("paging did not happen: no address dir under %s/paging (exit %d)\npaging dir:\n%s", instanceDataDir, code, top) + } + pages := 0 + for _, line := range strings.Fields(out) { + if strings.HasSuffix(line, ".page") { + pages++ + } + } + if pages < 2 { + t.Fatalf("paging produced %d page files (< 2); paging listing:\n%s", pages, out) + } + t.Logf("paging OK: %d page files", pages) +} + +// copyVolumeOut runs a busybox helper container that mounts the named volume +// read-only plus a host bind dir and copies the volume's contents out. It +// returns the host dir holding the copied data/. +func copyVolumeOut(t *testing.T, ctx context.Context, vol string) string { + t.Helper() + out := t.TempDir() + helperReq := tc.ContainerRequest{ + Image: "busybox:1.36", + // cp -r (not -a): under rootless podman the helper's root maps to the + // host user, but -a would preserve the artemis UID (an unmapped + // subuid), leaving files the host user cannot delete when t.TempDir + // cleans up. a+rwX keeps everything readable and removable. + Cmd: []string{"sh", "-c", "cp -r /vol/. /out/ && chmod -R a+rwX /out"}, + HostConfigModifier: func(hc *container.HostConfig) { + // :Z relabels the bind for SELinux hosts (Fedora); harmless elsewhere. + hc.Binds = append(hc.Binds, vol+":/vol:ro", out+":/out:Z") + }, + WaitingFor: wait.ForExit().WithExitTimeout(60 * time.Second), + } + helper, err := tc.GenericContainer(ctx, tc.GenericContainerRequest{ContainerRequest: helperReq, Started: true}) + if err != nil { + t.Fatalf("run copy helper: %v", err) + } + defer helper.Terminate(context.Background()) + state, err := helper.State(ctx) + if err != nil { + t.Fatalf("helper state: %v", err) + } + if state.ExitCode != 0 { + rd, lerr := helper.Logs(ctx) + logs := "" + if lerr == nil { + b, _ := io.ReadAll(rd) + logs = string(b) + } + t.Fatalf("copy helper exited %d; logs:\n%s", state.ExitCode, logs) + } + // Sanity: the copy must contain the journal dir. + if _, err := os.Stat(filepath.Join(out, "journal")); err != nil { + entries, _ := os.ReadDir(out) + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + t.Fatalf("copied volume has no journal/ (contents: %v): %v", names, err) + } + return out +} + +// tarGzDir packs srcDir into a gzipped tar at dest, with all paths rooted at +// root/ (e.g. "data/journal/activemq-data-1.amq"). +func tarGzDir(srcDir, root, dest string) error { + f, err := os.Create(dest) + if err != nil { + return err + } + defer f.Close() + gz, err := gzip.NewWriterLevel(f, gzip.BestCompression) + if err != nil { + return err + } + tw := tar.NewWriter(gz) + + err = filepath.Walk(srcDir, func(path string, info os.FileInfo, werr error) error { + if werr != nil { + return werr + } + rel, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + name := root + if rel != "." { + name = root + "/" + filepath.ToSlash(rel) + } + hdr, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + hdr.Name = name + if info.IsDir() { + hdr.Name += "/" + } + // Normalize ownership for a reproducible, committable fixture. + hdr.Uid, hdr.Gid = 0, 0 + hdr.Uname, hdr.Gname = "", "" + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if info.IsDir() || !info.Mode().IsRegular() { + return nil + } + src, err := os.Open(path) + if err != nil { + return err + } + defer src.Close() + _, err = io.Copy(tw, src) + return err + }) + if err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + if err := gz.Close(); err != nil { + return err + } + return f.Close() +} + +// mustModuleDir returns the repo root (the dir containing go.mod), so the +// harvester writes testdata/ at a stable path regardless of test cwd. +func mustModuleDir(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("go.mod not found above test dir") + } + dir = parent + } +} diff --git a/internal/journal/large.go b/internal/journal/large.go new file mode 100644 index 0000000..2bc85df --- /dev/null +++ b/internal/journal/large.go @@ -0,0 +1,110 @@ +package journal + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" +) + +// LargeDiag reports large-message joining problems. +type LargeDiag struct { + MissingFile []int64 // messageIDs whose .msg is absent (skip + report) + Orphans []string // .msg files with no surviving journal record (normal; report only) + LargestBytes int64 // size of the largest successfully attached large-message body +} + +// AttachLargeBodies joins each msg.Large message with its +// /.msg body, producing the complete raw AMQP bytes in msg.AMQP. +// +// format_notes.md section 5b (fixture-verified): "data/large-messages/.msg +// holds the complete AMQP-encoded message (Header + Properties + ... + Data +// section with the full body) -- fixture 64.msg is 307255 bytes = 55 bytes of +// sections + 307200 body, and it starts with 00 53 70 (Header descriptor), +// not raw body bytes. So msg.AMQP = the .msg file bytes verbatim; the journal +// record's saved-encoding block is only a section index / header +// cross-check." This is the opposite of the brief's fallback branch (journal +// header + file body concatenation) -- that branch does not apply here. +// +// Messages whose file is missing are removed from the returned slice and +// their ID recorded in LargeDiag.MissingFile. Non-Large messages pass through +// unchanged. Bodies are assembled in RAM (spec: accepted v1 cost). +// +// Strictly read-only on dir: only os.ReadDir/os.ReadFile are used. +func AttachLargeBodies(msgs []Message, dir string) ([]Message, LargeDiag, error) { + var diag LargeDiag + + entries, err := os.ReadDir(dir) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, diag, fmt.Errorf("journal: read large-messages dir %s: %w", dir, err) + } + // Missing large-messages/ directory is acceptable (no large bodies to attach). + // Large messages will be dropped when their individual .msg files are not found. + entries = nil + } + + // referenced tracks which .msg files are actually claimed by a + // surviving Large message, so any leftover *.msg is reported as an + // orphan (format_notes.md: normal, not an error -- the large message's + // journal record may itself have been acked/deleted while the body file + // lingers on disk). + referenced := make(map[string]bool) + for _, m := range msgs { + if isLarge, _ := m.largeBodyTarget(); isLarge { + referenced[strconv.FormatInt(m.ID, 10)+".msg"] = true + } + } + + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + // Non-.msg junk (partial writes, stray files) is silently ignored -- + // only *.msg files are large-message bodies (brief Step: "decide + // handling (ignore silently vs orphan-report) and document"). + if !strings.HasSuffix(name, ".msg") { + continue + } + if !referenced[name] { + diag.Orphans = append(diag.Orphans, name) + } + } + + out := make([]Message, 0, len(msgs)) + for _, m := range msgs { + isLarge, isCore := m.largeBodyTarget() + if !isLarge { + out = append(out, m) + continue + } + + path := filepath.Join(dir, strconv.FormatInt(m.ID, 10)+".msg") + body, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + diag.MissingFile = append(diag.MissingFile, m.ID) + continue + } + return nil, diag, fmt.Errorf("journal: read large-message body %s: %w", path, err) + } + + // AMQP large bodies are the complete AMQP-encoded message (msg.AMQP); + // Core large bodies are the raw message body bytes (msg.Core.Body). + if isCore { + m.Core.Body = body + } else { + m.AMQP = body + } + if int64(len(body)) > diag.LargestBytes { + diag.LargestBytes = int64(len(body)) + } + out = append(out, m) + } + + return out, diag, nil +} diff --git a/internal/journal/large_test.go b/internal/journal/large_test.go new file mode 100644 index 0000000..b8638b3 --- /dev/null +++ b/internal/journal/large_test.go @@ -0,0 +1,286 @@ +package journal + +import ( + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "testing" + + "github.com/Azure/go-amqp" +) + +// copyDir recursively copies src into dst (both must exist/be creatable); +// used to get a mutable copy of the fixture's large-messages dir for the +// delete/orphan tests, since fixtureDir's extraction must stay read-only per +// this task's "strictly read-only on the source dir" contract. +func copyDir(t *testing.T, src, dst string) { + t.Helper() + entries, err := os.ReadDir(src) + if err != nil { + t.Fatalf("copyDir: read %s: %v", src, err) + } + for _, e := range entries { + srcPath := filepath.Join(src, e.Name()) + dstPath := filepath.Join(dst, e.Name()) + if e.IsDir() { + if err := os.MkdirAll(dstPath, 0o755); err != nil { + t.Fatalf("copyDir: mkdir %s: %v", dstPath, err) + } + copyDir(t, srcPath, dstPath) + continue + } + in, err := os.Open(srcPath) + if err != nil { + t.Fatalf("copyDir: open %s: %v", srcPath, err) + } + out, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + in.Close() + t.Fatalf("copyDir: create %s: %v", dstPath, err) + } + if _, err := io.Copy(out, in); err != nil { + in.Close() + out.Close() + t.Fatalf("copyDir: copy %s: %v", srcPath, err) + } + in.Close() + if err := out.Close(); err != nil { + t.Fatalf("copyDir: close %s: %v", dstPath, err) + } + } +} + +// fixtureLargeMessage returns the salvage.large journal-decoded Message +// (Large=true, AMQP empty) from the full fixture pipe, plus the fixture's +// large-messages dir path. +func fixtureLargeMessage(t *testing.T) (Message, string) { + t.Helper() + dir := fixtureDir(t) + messages, _, names := decodeFixtureMessages(t) + byQ := byQueue(t, messages, names) + got := byQ["salvage.large"] + if len(got) != 1 { + t.Fatalf("want 1 salvage.large message, got %d", len(got)) + } + return got[0], filepath.Join(dir, "large-messages") +} + +func TestAttachLargeBodiesFixtureRoundTrip(t *testing.T) { + msg, largeDir := fixtureLargeMessage(t) + man := loadManifest(t) + want := man.Queues["salvage.large"] + if len(want) != 1 { + t.Fatalf("manifest: want 1 salvage.large entry, got %d", len(want)) + } + + out, diag, err := AttachLargeBodies([]Message{msg}, largeDir) + if err != nil { + t.Fatalf("AttachLargeBodies: %v", err) + } + if len(out) != 1 { + t.Fatalf("want 1 message, got %d", len(out)) + } + if len(diag.MissingFile) != 0 { + t.Errorf("MissingFile = %v, want empty", diag.MissingFile) + } + if len(diag.Orphans) != 0 { + t.Errorf("Orphans = %v, want empty", diag.Orphans) + } + + // format_notes.md section 5b: the .msg file is the complete AMQP-encoded + // message (Header...Data...), not just the body -- so msg.AMQP must be + // the file bytes verbatim, round-trippable via go-amqp's UnmarshalBinary, + // and its Data section must match the manifest's sha256/len. + var am amqp.Message + if err := am.UnmarshalBinary(out[0].AMQP); err != nil { + t.Fatalf("unmarshal joined AMQP: %v", err) + } + body := am.GetData() + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + if hash != want[0].BodySha256 { + t.Errorf("body sha256 = %s, want %s", hash, want[0].BodySha256) + } + if len(body) != want[0].BodyLen { + t.Errorf("body len = %d, want %d", len(body), want[0].BodyLen) + } + if len(body) != 307200 { + t.Errorf("body len = %d, want 307200 (manifest-documented)", len(body)) + } + + if diag.LargestBytes < 307200 { + t.Errorf("LargestBytes = %d, want >= 307200", diag.LargestBytes) + } +} + +func TestAttachLargeBodiesMissingFileDropsMessage(t *testing.T) { + msg, largeDir := fixtureLargeMessage(t) + + tmpDir := t.TempDir() + copyDir(t, largeDir, tmpDir) + if err := os.Remove(filepath.Join(tmpDir, "64.msg")); err != nil { + t.Fatalf("remove fixture copy's 64.msg: %v", err) + } + + out, diag, err := AttachLargeBodies([]Message{msg}, tmpDir) + if err != nil { + t.Fatalf("AttachLargeBodies: %v", err) + } + if len(out) != 0 { + t.Fatalf("want message dropped, got %d: %+v", len(out), out) + } + if len(diag.MissingFile) != 1 || diag.MissingFile[0] != msg.ID { + t.Errorf("MissingFile = %v, want [%d]", diag.MissingFile, msg.ID) + } + if len(diag.Orphans) != 0 { + t.Errorf("Orphans = %v, want empty", diag.Orphans) + } +} + +func TestAttachLargeBodiesOrphanFileReported(t *testing.T) { + msg, largeDir := fixtureLargeMessage(t) + + tmpDir := t.TempDir() + copyDir(t, largeDir, tmpDir) + orphanPath := filepath.Join(tmpDir, "999999.msg") + if err := os.WriteFile(orphanPath, []byte("orphan body"), 0o644); err != nil { + t.Fatalf("write orphan file: %v", err) + } + + out, diag, err := AttachLargeBodies([]Message{msg}, tmpDir) + if err != nil { + t.Fatalf("AttachLargeBodies: %v", err) + } + if len(out) != 1 { + t.Fatalf("want 1 message (unaffected by orphan), got %d", len(out)) + } + if len(diag.MissingFile) != 0 { + t.Errorf("MissingFile = %v, want empty", diag.MissingFile) + } + if len(diag.Orphans) != 1 || diag.Orphans[0] != "999999.msg" { + t.Errorf("Orphans = %v, want [999999.msg]", diag.Orphans) + } + + // The surviving message's own body must be unaffected by the orphan. + var am amqp.Message + if err := am.UnmarshalBinary(out[0].AMQP); err != nil { + t.Fatalf("unmarshal joined AMQP: %v", err) + } + if len(am.GetData()) != 307200 { + t.Errorf("body len = %d, want 307200", len(am.GetData())) + } +} + +func TestAttachLargeBodiesIgnoresNonMsgJunk(t *testing.T) { + msg, largeDir := fixtureLargeMessage(t) + + tmpDir := t.TempDir() + copyDir(t, largeDir, tmpDir) + // Non-.msg junk (e.g. a stray .tmp from an interrupted write, or a + // directory) must not be reported as an orphan or otherwise disturb the + // result -- only *.msg files are large-message bodies. + if err := os.WriteFile(filepath.Join(tmpDir, "notes.txt"), []byte("junk"), 0o644); err != nil { + t.Fatalf("write junk file: %v", err) + } + if err := os.Mkdir(filepath.Join(tmpDir, "subdir"), 0o755); err != nil { + t.Fatalf("mkdir junk subdir: %v", err) + } + + out, diag, err := AttachLargeBodies([]Message{msg}, tmpDir) + if err != nil { + t.Fatalf("AttachLargeBodies: %v", err) + } + if len(out) != 1 { + t.Fatalf("want 1 message, got %d", len(out)) + } + if len(diag.MissingFile) != 0 { + t.Errorf("MissingFile = %v, want empty", diag.MissingFile) + } + if len(diag.Orphans) != 0 { + t.Errorf("Orphans = %v, want empty (non-.msg files are silently ignored)", diag.Orphans) + } +} + +func TestAttachLargeBodiesEmptyInput(t *testing.T) { + out, diag, err := AttachLargeBodies(nil, t.TempDir()) + if err != nil { + t.Fatalf("AttachLargeBodies: %v", err) + } + if len(out) != 0 { + t.Errorf("out = %v, want empty", out) + } + if len(diag.MissingFile) != 0 || len(diag.Orphans) != 0 || diag.LargestBytes != 0 { + t.Errorf("diag = %+v, want zero", diag) + } +} + +// TestAttachLargeBodiesLeavesNonLargeMessagesUntouched: a non-Large message +// passed through must survive unchanged (defensive -- callers are expected +// to filter to Large-only, but the function should not corrupt other +// messages if handed a mixed slice). +func TestAttachLargeBodiesLeavesNonLargeMessagesUntouched(t *testing.T) { + plain := Message{ID: 999, AMQP: []byte("already-here"), QueueIDs: []int64{1}} + + out, diag, err := AttachLargeBodies([]Message{plain}, t.TempDir()) + if err != nil { + t.Fatalf("AttachLargeBodies: %v", err) + } + if len(out) != 1 { + t.Fatalf("want 1 message, got %d", len(out)) + } + if string(out[0].AMQP) != "already-here" { + t.Errorf("AMQP = %q, want unchanged", out[0].AMQP) + } + if len(diag.MissingFile) != 0 || len(diag.Orphans) != 0 { + t.Errorf("diag = %+v, want zero", diag) + } +} + +// TestAttachLargeBodiesNonexistentDirWithLargeMessage: a missing large-messages/ +// directory is acceptable (not fatal). Large messages are dropped (file not found) +// and recorded in MissingFile. +func TestAttachLargeBodiesNonexistentDirWithLargeMessage(t *testing.T) { + largeMsg := Message{ID: 64, Large: true, QueueIDs: []int64{1}} + nonexistentDir := filepath.Join(t.TempDir(), "nonexistent-large-messages") + + out, diag, err := AttachLargeBodies([]Message{largeMsg}, nonexistentDir) + if err != nil { + t.Fatalf("AttachLargeBodies: %v", err) + } + if len(out) != 0 { + t.Fatalf("want message dropped, got %d: %+v", len(out), out) + } + if len(diag.MissingFile) != 1 || diag.MissingFile[0] != 64 { + t.Errorf("MissingFile = %v, want [64]", diag.MissingFile) + } + if len(diag.Orphans) != 0 { + t.Errorf("Orphans = %v, want empty", diag.Orphans) + } +} + +// TestAttachLargeBodiesNonexistentDirWithNonLargeMessages: a missing large-messages/ +// directory is acceptable. Non-Large messages pass through unchanged with empty diag. +func TestAttachLargeBodiesNonexistentDirWithNonLargeMessages(t *testing.T) { + msg1 := Message{ID: 100, AMQP: []byte("msg1-body"), QueueIDs: []int64{1}} + msg2 := Message{ID: 200, AMQP: []byte("msg2-body"), QueueIDs: []int64{1}} + nonexistentDir := filepath.Join(t.TempDir(), "nonexistent-large-messages") + + out, diag, err := AttachLargeBodies([]Message{msg1, msg2}, nonexistentDir) + if err != nil { + t.Fatalf("AttachLargeBodies: %v", err) + } + if len(out) != 2 { + t.Fatalf("want 2 messages, got %d", len(out)) + } + if string(out[0].AMQP) != "msg1-body" { + t.Errorf("out[0].AMQP = %q, want unchanged", out[0].AMQP) + } + if string(out[1].AMQP) != "msg2-body" { + t.Errorf("out[1].AMQP = %q, want unchanged", out[1].AMQP) + } + if len(diag.MissingFile) != 0 || len(diag.Orphans) != 0 || diag.LargestBytes != 0 { + t.Errorf("diag = %+v, want zero", diag) + } +} diff --git a/internal/journal/message.go b/internal/journal/message.go new file mode 100644 index 0000000..a2c4f81 --- /dev/null +++ b/internal/journal/message.go @@ -0,0 +1,465 @@ +package journal + +import "sort" + +// Persister ids (PersisterIDs), format_notes.md section 5. Verified against +// apache/activemq-artemis tag 2.42.0 PersisterIDs.java: MAX_PERSISTERS = 5, +// MessagePersister.getPersister(id) = persisters[id-1] (id 0 or > 5 is +// invalid). The persister id is the first byte of an ADD_MESSAGE_PROTOCOL +// (userType 45) record's body. NOTE this corrects the plan's guessed +// numbering (V3 is 5, not 4 -- AMQPLargeMessagePersister is 4). +const ( + // persisterCoreLargeMessage (id 0) documents the full PersisterIDs table + // for reference; it never appears on an ADD_MESSAGE_PROTOCOL record -- + // core large messages use userType 30 instead (see DecodeMessages). + persisterCoreLargeMessage byte = 0 + persisterCoreMessage byte = 1 + persisterAMQPMessage byte = 2 + persisterAMQPMessageV2 byte = 3 + persisterAMQPLargeMessage byte = 4 + persisterAMQPMessageV3 byte = 5 +) + +// Core message body type bytes (org.apache.activemq.artemis.api.core.Message +// TEXT_TYPE/BYTES_TYPE/... — verified against a real 2.42.0 record, which is a +// BYTES message, type 4). DEFAULT is the untyped core message. +const ( + CoreTypeDefault byte = 0 + CoreTypeObject byte = 2 + CoreTypeText byte = 3 + CoreTypeBytes byte = 4 + CoreTypeMap byte = 5 + CoreTypeStream byte = 6 +) + +// Core buffer framing constants, verified byte-for-byte against a real 2.42.0 +// broker record (testdata/core-record-2.42.bin); see format_notes.md §5c. +// coreBufferHeaderSpace = PacketImpl.PACKET_HEADERS_SIZE (SIZE_INT + SIZE_BYTE +// + SIZE_LONG); coreBodyOffset = DataConstants.SIZE_INT. +const ( + coreBufferHeaderSpace = 13 + coreBodyOffset = 4 +) + +// CorePayload is a decoded Artemis Core-protocol message, either a standard +// message (journal userType 45 / persister id 1, or a persister-1 page entry) +// or a Core large message (userType 30, whose Body is joined from +// data/large-messages/.msg). It is what a Core store record carries so it +// can be dumped and later converted to AMQP for redelivery (broker/coreconvert). +type CorePayload struct { + MessageID int64 + Address string + UserID []byte // 16-byte UUID, or nil when absent + Type byte // CoreType* — 0 DEFAULT, 3 TEXT, 4 BYTES, 5 MAP, 2 OBJECT, 6 STREAM + Durable bool + Expiration int64 + Timestamp int64 + Priority byte + Properties map[string]any // decoded TypedProperties (may be nil) + Body []byte // message body bytes; empty for Large until joined + Large bool +} + +// decodeCoreHeaders decodes a CoreMessage encodeHeadersAndProperties block +// (format_notes.md §5c): long messageID, nullableSimpleString address, a +// userID null-flag byte (+16 bytes when NOT_NULL), byte type, boolean durable, +// long expiration, long timestamp, byte priority, then TypedProperties. It +// returns nil if the block runs past the available bytes. Shared by the +// standard-message decoder (after its endOfBodyPosition/body prefix) and the +// Core large decoder (which is this block verbatim, body-in-.msg). +func decodeCoreHeaders(r *reader) *CorePayload { + p := &CorePayload{} + p.MessageID = r.i64() + addr, _ := r.nullableSimpleString() + p.Address = addr + flag := r.u8() + if r.err() != nil { + return nil + } + if flag == dcNotNull { + p.UserID = append([]byte(nil), r.bytes(16)...) + } + p.Type = r.u8() + p.Durable = r.bool() + p.Expiration = r.i64() + p.Timestamp = r.i64() + p.Priority = r.u8() + p.Properties = r.typedProperties() + if r.err() != nil { + return nil + } + return p +} + +// decodeCoreStandardBody decodes a CoreMessagePersister (id 1) payload, with r +// positioned just past the persister-id byte (format_notes.md §5c): a long +// messageID and nullableSimpleString address prefix (both redundant with the +// authoritative copies inside the headers block, so read-and-discarded), an +// int bufferSize, then the CoreMessage buffer whose first int is +// endOfBodyPosition. The body is buffer[coreBodyOffset : endOfBodyPosition - +// coreBufferHeaderSpace + coreBodyOffset]; the headers block follows. Returns +// nil on any truncation. +func decodeCoreStandardBody(r *reader) *CorePayload { + r.i64() // messageID prefix — authoritative copy is in the headers + r.nullableSimpleString() // address prefix — ditto + r.i32() // bufferSize (message.persist length prefix); body/headers self-delimit + if r.err() != nil { + return nil + } + endOfBody := int(r.i32()) // CoreMessage buffer[0..4): endOfBodyPosition + if r.err() != nil { + return nil + } + bodyLen := endOfBody - coreBufferHeaderSpace + if bodyLen < 0 { + return nil + } + body := append([]byte(nil), r.bytes(bodyLen)...) + if r.err() != nil { + return nil + } + p := decodeCoreHeaders(r) + if p == nil { + return nil + } + p.Body = body + return p +} + +// Message is one decoded, surviving journal message, ready to be written to +// the .artx store (Large messages still need their body joined from +// data/large-messages/.msg by Task 8). +type Message struct { + ID int64 + AMQP []byte // raw AMQP wire bytes, verbatim from the persister payload; empty for Large or Core messages + Core *CorePayload // non-nil for a decoded Core-protocol message (AMQP is empty); Core.Large marks a body joined from large-messages dir + Large bool // AMQP large message: body joined from large-messages dir (Task 8); AMQP is empty here, not partial + ScheduledMs int64 // 0 = none; from a SET_SCHEDULED_DELIVERY_TIME update + QueueIDs []int64 // surviving refs (ADD_REF minus ACKNOWLEDGE_REF), first-seen order +} + +// largeBodyTarget reports whether m needs its body joined from the +// large-messages dir, and whether it is a Core (vs AMQP) large message. +func (m Message) largeBodyTarget() (isLarge, isCore bool) { + if m.Core != nil && m.Core.Large { + return true, true + } + return m.Large, false +} + +// MessageDiag itemizes skips per spec §1/§5. +type MessageDiag struct { + CoreSkipped map[int64][]int64 // messageID → surviving queueIDs, for Core-protocol messages (userType 31/30, or persister id 1 under userType 45) this AMQP-only reader cannot decode + UnknownPersister int // ADD_MESSAGE_PROTOCOL records whose first byte isn't a recognized persister id + UndecodableBody int // records whose body was too short/malformed to parse past the point a persister id was identified +} + +// DecodeMessages walks message-journal survivors and joins messages with +// their refs. Messages (and Core-skipped entries) with zero surviving refs +// are dropped entirely -- format_notes.md section 9's removeAcked: a message +// with no remaining queue refs has already been fully consumed, so there is +// nothing left to salvage or usefully report as skipped. +// +// Design note (page-cursor / non-message records, and the brief's +// CursorState question): the message journal interleaves message survivors +// (ADD_MESSAGE_PROTOCOL/ADD_MESSAGE/ADD_LARGE_MESSAGE, userTypes 45/31/30) +// with unrelated record families that replay through the same Replayer: +// PAGE_TRANSACTION (35) and the PAGE_CURSOR_*/ACKNOWLEDGE_CURSOR family +// (39-43), each journaled as its own top-level survivor with its own record +// ID (not as an update riding on a message ID). DecodeMessages' contract is +// message decode only: it recognizes exactly the three message-family +// userTypes above and silently ignores every other survivor -- no +// CursorRecords helper, no filtering of the input slice. Task 9 re-scans the +// same []Survivor returned by Replayer.Resolve for its own record families. +// This is the simpler of the brief's two options: it costs nothing here (a +// no-op default case) and avoids threading a second typed accessor through +// this package for state DecodeMessages never touches. +// +// error is reserved for future use (e.g. a structural invariant violation); +// per-record decode failures are diagnostic counters, not errors, so every +// call currently returns a nil error. +func DecodeMessages(survivors []Survivor) ([]Message, MessageDiag, error) { + diag := MessageDiag{CoreSkipped: make(map[int64][]int64)} + var out []Message + + for _, sv := range survivors { + switch sv.UserType { + case AddMessageProtocol: + msg, outcome := decodeAddMessageProtocol(sv) + switch outcome { + case outcomeUndecodable: + diag.UndecodableBody++ + case outcomeUnknownPersister: + diag.UnknownPersister++ + case outcomeCore, outcomeMessage: + // Both AMQP and decoded-Core messages are exported; the Message + // carries either AMQP bytes or a *CorePayload. + if len(msg.QueueIDs) > 0 { + out = append(out, msg) + } + } + + case AddLargeMessage: + // Core large message (userType 30, format_notes.md §5c): its body is + // the encodeHeadersAndProperties block directly (no persister-id / + // endOfBodyPosition prefix), and the message body lives in + // data/large-messages/.msg (joined by AttachLargeBodies). + queueIDs, scheduledMs := decodeRefs(sv.Updates) + core := decodeCoreHeaders(newReader(sv.Body)) + if core == nil { + // Undecodable core-large header: still report the skip per queue. + if len(queueIDs) > 0 { + diag.CoreSkipped[sv.ID] = queueIDs + } + continue + } + core.Large = true + if len(queueIDs) > 0 { + out = append(out, Message{ID: sv.ID, Core: core, ScheduledMs: scheduledMs, QueueIDs: queueIDs}) + } + + case AddMessage: + // Legacy pre-persister-id core add (userType 31): not produced by + // Artemis 2.42 (which writes core messages as userType 45 / persister + // 1), unverified against real bytes, so still reported as a skip + // rather than best-effort decoded. Walk the refs to attribute it. + queueIDs, _ := decodeRefs(sv.Updates) + if len(queueIDs) > 0 { + diag.CoreSkipped[sv.ID] = queueIDs + } + + default: + // Not a message record -- see the design note above. + } + } + + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, diag, nil +} + +// decodeOutcome classifies how decodeAddMessageProtocol resolved one +// ADD_MESSAGE_PROTOCOL survivor. +type decodeOutcome int + +const ( + outcomeMessage decodeOutcome = iota + outcomeCore + outcomeUnknownPersister + outcomeUndecodable +) + +// decodeAddMessageProtocol decodes one ADD_MESSAGE_PROTOCOL (userType 45) +// survivor via the shared decodePersisterPayload (format_notes.md section 5: +// the body's first byte is the persister id, dispatching to Core (skip), +// AMQP standard (2/3/5), or AMQP large (4)). +func decodeAddMessageProtocol(sv Survivor) (Message, decodeOutcome) { + r := newReader(sv.Body) + amqpBytes, core, _, outcome := decodePersisterPayload(r) + queueIDs, scheduledMs := decodeRefs(sv.Updates) + + switch outcome { + case persisterDecodeCore: + return Message{ID: sv.ID, Core: core, ScheduledMs: scheduledMs, QueueIDs: queueIDs}, outcomeCore + + case persisterDecodeLarge: + // format_notes.md section 5b: the large body itself is not in the + // journal (it lives in data/large-messages/.msg, joined by + // Task 8); nothing else in this record (durable/format/address/extra + // props/saved-encoding block) is needed for Task 7's contract. + return Message{ID: sv.ID, Large: true, ScheduledMs: scheduledMs, QueueIDs: queueIDs}, outcomeMessage + + case persisterDecodeStandard: + return Message{ID: sv.ID, AMQP: amqpBytes, ScheduledMs: scheduledMs, QueueIDs: queueIDs}, outcomeMessage + + case persisterDecodeUnknown: + return Message{}, outcomeUnknownPersister + + default: // persisterDecodeMalformed + return Message{}, outcomeUndecodable + } +} + +// persisterDecodeOutcome classifies how decodePersisterPayload resolved a +// persister-tagged payload -- shared by message.go's ADD_MESSAGE_PROTOCOL +// bodies (format_notes.md section 5) and paging.go's PagedMessage entries +// (format_notes.md section 8), which both lead with a persister-id byte +// dispatching to the same cases. +type persisterDecodeOutcome int + +const ( + // persisterDecodeStandard: AMQP standard (2/3/5). amqpBytes is + // populated; r is left positioned just past the raw AMQP bytes -- the + // V2/V3 tail (extra properties, V3's expiration) is NOT consumed, see + // decodePersisterPayload's doc comment. + persisterDecodeStandard persisterDecodeOutcome = iota + // persisterDecodeLarge: AMQP large (4). The body lives outside this + // payload entirely (data/large-messages/.msg for journal-resident + // messages; out of scope for paged entries, format_notes.md section 8). + persisterDecodeLarge + // persisterDecodeCore: Core message (persister 1), decoded into the + // returned *CorePayload (format_notes.md §5c). + persisterDecodeCore + // persisterDecodeUnknown: an unrecognized persister-id byte. + persisterDecodeUnknown + // persisterDecodeMalformed: a recognized id, but the fixed prefix ran + // past the available bytes. + persisterDecodeMalformed +) + +// decodePersisterPayload decodes a MessagePersister-encoded payload's +// leading dispatch (format_notes.md section 5's PersisterIDs table): a +// persister-id byte, then -- for the AMQP standard cases (2/3/5) only -- the +// fixed prefix through the raw AMQP bytes (format_notes.md section 5a: +// messageID, messageFormat, nullableSimpleString address, int amqpSize, +// amqpSize raw AMQP bytes). It reads from r in place (rather than taking a +// []byte) so a caller that needs to keep parsing past the payload -- +// paging.go's PagedMessage, which has queueIDs following it -- can continue +// from the same cursor; a caller with nothing left to read (this file's +// ADD_MESSAGE_PROTOCOL body) simply stops. +// +// It deliberately does NOT consume the V2/V3 tail (int extraPropsSize + +// that many bytes of TypedProperties, V3's long expiration) even though +// that tail is formally part of "the persister payload" in the Artemis +// source: neither this file's caller needs its content, and the existing +// synthetic tests (buildAMQPStandardBody in message_test.go) don't encode +// it. paging.go, which DOES need the exact end offset to find the +// queueIDsCount field that follows, skips that tail itself using the +// returned persisterID (see paging.go's skipPersisterTail). +// +// Deviation from the task-9 brief: the brief specified +// `decodePersisterPayload(b []byte) (amqpBytes []byte, scheduledMs int64, +// core bool, err error)`. scheduledMs is dropped entirely -- it was never +// part of either caller's persister payload (this file's comes from a +// separate SET_SCHEDULED_DELIVERY_TIME update, decodeRefs below; a +// PagedMessage has no scheduled-delivery field on disk at all, per +// format_notes.md section 8's field table). `core bool, err error` are +// widened to `persisterID byte, persisterDecodeOutcome` so both callers keep +// Task 7's existing unknown-persister/malformed-body diagnostic granularity +// (and paging.go gets the persister id it needs for the tail skip) instead +// of collapsing everything to one generic error. The []byte parameter +// became a *reader for the in-place-continuation reason above. +func decodePersisterPayload(r *reader) (amqpBytes []byte, core *CorePayload, persisterID byte, outcome persisterDecodeOutcome) { + persisterID = r.u8() + if r.err() != nil { + return nil, nil, 0, persisterDecodeMalformed + } + + switch persisterID { + case persisterCoreMessage: + core = decodeCoreStandardBody(r) + if core == nil { + return nil, nil, persisterID, persisterDecodeMalformed + } + return nil, core, persisterID, persisterDecodeCore + + case persisterAMQPLargeMessage: + return nil, nil, persisterID, persisterDecodeLarge + + case persisterAMQPMessage, persisterAMQPMessageV2, persisterAMQPMessageV3: + amqp, ok := decodeAMQPStandardBody(r) + if !ok { + return nil, nil, persisterID, persisterDecodeMalformed + } + return amqp, nil, persisterID, persisterDecodeStandard + + default: + return nil, nil, persisterID, persisterDecodeUnknown + } +} + +// decodeAMQPStandardBody decodes the remainder of an AMQPMessagePersister / +// V2 / V3 record after the persister id byte (format_notes.md section 5a): +// long messageID, long messageFormat, nullableSimpleString address, int +// amqpSize, then amqpSize bytes of raw AMQP-encoded message. The AMQP body +// is length-prefixed, so the V2/V3 tail (extra properties, V3's expiration +// long) is unambiguous and simply left unread -- Task 7 only needs the raw +// AMQP bytes, and the message ID used to join refs is the journal record ID +// (Survivor.ID), not this redundant persister-internal copy. +func decodeAMQPStandardBody(r *reader) ([]byte, bool) { + r.i64() // messageID: redundant with Survivor.ID, not surfaced + r.i64() // messageFormat: not surfaced on Message + r.nullableSimpleString() // address: not surfaced (queue mapping comes from bindings + refs) + amqpSize := r.i32() + raw := r.bytes(int(amqpSize)) + if r.err() != nil { + return nil, false + } + amqpBytes := make([]byte, len(raw)) + copy(amqpBytes, raw) + return amqpBytes, true +} + +// decodeRefs nets a message's surviving queue refs and scheduled delivery +// time out of its Updates (format_notes.md section 9): ADD_REF (32) +// increments a queueID's count, ACKNOWLEDGE_REF (33) decrements it, and only +// queueIDs with a positive count survive -- mirroring removeAcked's "no +// remaining refs" elimination without relying on the broker having also +// deleted the message record outright. SET_SCHEDULED_DELIVERY_TIME (36) +// updates set scheduledMs (ScheduledDeliveryEncoding: long queueID, long +// scheduledDeliveryTime -- the queueID is not surfaced on Message, which +// carries a single scheduled time regardless of queue count). Every other +// update userType (UPDATE_DELIVERY_COUNT, DUPLICATE_ID, ACK_RETRY, ...) is +// ignored here per the task brief. A malformed ref/scheduled body (should +// not happen against real journal bytes) is skipped rather than treated as +// fatal, consistent with this reader's general resync-not-abort posture. +func decodeRefs(updates []RawRecord) (queueIDs []int64, scheduledMs int64) { + counts := make(map[int64]int) + var order []int64 // first-seen order, for a deterministic QueueIDs result + + for _, u := range updates { + switch u.UserType { + case AddRef: + qid, ok := decodeQueueID(u.Body) + if !ok { + continue + } + if _, seen := counts[qid]; !seen { + order = append(order, qid) + } + counts[qid]++ + + case AcknowledgeRef: + qid, ok := decodeQueueID(u.Body) + if !ok { + continue + } + counts[qid]-- + + case SetScheduledDeliveryTime: + _, ms, ok := decodeScheduledDelivery(u.Body) + if ok { + scheduledMs = ms + } + } + } + + for _, qid := range order { + if counts[qid] > 0 { + queueIDs = append(queueIDs, qid) + } + } + return queueIDs, scheduledMs +} + +// decodeQueueID decodes a RefEncoding body (format_notes.md section 9's +// "Supporting codec field orders": QueueEncoding.decode = long queueID). +func decodeQueueID(body []byte) (int64, bool) { + r := newReader(body) + qid := r.i64() + if r.err() != nil { + return 0, false + } + return qid, true +} + +// decodeScheduledDelivery decodes a ScheduledDeliveryEncoding body +// (format_notes.md section 9: long queueID, long scheduledDeliveryTime). +func decodeScheduledDelivery(body []byte) (queueID, scheduledDeliveryTime int64, ok bool) { + r := newReader(body) + queueID = r.i64() + scheduledDeliveryTime = r.i64() + if r.err() != nil { + return 0, 0, false + } + return queueID, scheduledDeliveryTime, true +} diff --git a/internal/journal/message_test.go b/internal/journal/message_test.go new file mode 100644 index 0000000..ee03228 --- /dev/null +++ b/internal/journal/message_test.go @@ -0,0 +1,758 @@ +package journal + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "unicode/utf16" + + "github.com/Azure/go-amqp" +) + +// --- synthetic byte-builders (mirror format_notes.md sections 5a/6/9's +// write-side layouts, the inverse of what message.go's readers expect) --- + +func beI32(v int32) []byte { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(v)) + return b +} + +func beI64(v int64) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(v)) + return b +} + +// encodeSimpleString mirrors format_notes.md section 6: big-endian int +// byteLength, then little-endian UTF-16 code-unit pairs. +func encodeSimpleString(s string) []byte { + units := utf16.Encode([]rune(s)) + data := make([]byte, len(units)*2) + for i, u := range units { + data[2*i] = byte(u) + data[2*i+1] = byte(u >> 8) + } + return append(beI32(int32(len(data))), data...) +} + +func encodeNullableSimpleString(s string, notNull bool) []byte { + if !notNull { + return []byte{0} + } + return append([]byte{1}, encodeSimpleString(s)...) +} + +// buildAMQPStandardBody builds an ADD_MESSAGE_PROTOCOL body for persister +// id in {2 (V1), 3 (V2), 5 (V3)}: format_notes.md section 5a's fixed prefix +// (messageID, messageFormat, nullableSimpleString address, int amqpSize, +// amqp bytes). The V2/V3 tail (extra props / expiration) is intentionally +// omitted -- decodeAMQPStandardBody never reads past the amqp bytes. +func buildAMQPStandardBody(persisterID byte, amqpBytes []byte) []byte { + body := []byte{persisterID} + body = append(body, beI64(42)...) // messageID (persister-internal, unused by the reader) + body = append(body, beI64(0)...) // messageFormat + body = append(body, encodeNullableSimpleString("test.address", true)...) + body = append(body, beI32(int32(len(amqpBytes)))...) + body = append(body, amqpBytes...) + return body +} + +func buildRefUpdate(msgID, queueID int64, userType byte) RawRecord { + return RawRecord{Type: UpdateRecord, ID: msgID, UserType: userType, Body: beI64(queueID)} +} + +func buildScheduledUpdate(msgID, queueID, ms int64) RawRecord { + return RawRecord{Type: UpdateRecord, ID: msgID, UserType: SetScheduledDeliveryTime, Body: append(beI64(queueID), beI64(ms)...)} +} + +// --- synthetic tests --- + +func TestDecodeMessagesAMQPStandardPersisters(t *testing.T) { + for _, persisterID := range []byte{persisterAMQPMessage, persisterAMQPMessageV2, persisterAMQPMessageV3} { + t.Run(fmt.Sprintf("persister_%d", persisterID), func(t *testing.T) { + amqpBytes := []byte("raw-amqp-bytes-verbatim") + sv := Survivor{ + ID: 10, + UserType: AddMessageProtocol, + Body: buildAMQPStandardBody(persisterID, amqpBytes), + Updates: []RawRecord{buildRefUpdate(10, 7, AddRef)}, + } + + messages, diag, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 1 { + t.Fatalf("want 1 message, got %d", len(messages)) + } + m := messages[0] + if m.ID != 10 { + t.Errorf("ID = %d, want 10", m.ID) + } + if string(m.AMQP) != string(amqpBytes) { + t.Errorf("AMQP = %q, want %q", m.AMQP, amqpBytes) + } + if m.Large { + t.Errorf("Large = true, want false") + } + if m.ScheduledMs != 0 { + t.Errorf("ScheduledMs = %d, want 0", m.ScheduledMs) + } + if len(m.QueueIDs) != 1 || m.QueueIDs[0] != 7 { + t.Errorf("QueueIDs = %v, want [7]", m.QueueIDs) + } + if diag.UnknownPersister != 0 || diag.UndecodableBody != 0 || len(diag.CoreSkipped) != 0 { + t.Errorf("diag = %+v, want zero", diag) + } + }) + } +} + +func TestDecodeMessagesLargePersister(t *testing.T) { + sv := Survivor{ + ID: 11, + UserType: AddMessageProtocol, + Body: []byte{persisterAMQPLargeMessage}, // decodeAddMessageProtocol reads nothing past the persister id for the large case + Updates: []RawRecord{buildRefUpdate(11, 3, AddRef)}, + } + + messages, diag, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 1 { + t.Fatalf("want 1 message, got %d", len(messages)) + } + m := messages[0] + if !m.Large { + t.Errorf("Large = false, want true") + } + if len(m.AMQP) != 0 { + t.Errorf("AMQP = %v, want empty (joined later by Task 8)", m.AMQP) + } + if len(m.QueueIDs) != 1 || m.QueueIDs[0] != 3 { + t.Errorf("QueueIDs = %v, want [3]", m.QueueIDs) + } + if diag.UnknownPersister != 0 || diag.UndecodableBody != 0 || len(diag.CoreSkipped) != 0 { + t.Errorf("diag = %+v, want zero", diag) + } +} + +func TestDecodeMessagesUnknownPersisterCounted(t *testing.T) { + sv := Survivor{ + ID: 12, + UserType: AddMessageProtocol, + Body: []byte{99}, // not a recognized persister id + Updates: []RawRecord{buildRefUpdate(12, 1, AddRef)}, + } + + messages, diag, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 0 { + t.Fatalf("want 0 messages, got %d: %+v", len(messages), messages) + } + if diag.UnknownPersister != 1 { + t.Errorf("diag.UnknownPersister = %d, want 1", diag.UnknownPersister) + } + if len(diag.CoreSkipped) != 0 { + t.Errorf("diag.CoreSkipped = %v, want empty", diag.CoreSkipped) + } +} + +// loadCoreRecord reads the golden Core persister payload harvested from a real +// Artemis 2.42.0 broker (an ADD_MESSAGE_PROTOCOL record's data: a BYTES +// message, messageID 28, address "salvage.core", 120-byte body). It is the +// ground truth for the Core decoder (format_notes.md §5c). +func loadCoreRecord(t *testing.T) []byte { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", "core-record-2.42.bin")) + if err != nil { + t.Fatalf("read core golden record: %v", err) + } + return b +} + +func TestDecodeMessagesCoreStandardExported(t *testing.T) { + sv := Survivor{ + ID: 13, + UserType: AddMessageProtocol, + Body: loadCoreRecord(t), + Updates: []RawRecord{buildRefUpdate(13, 4, AddRef), buildRefUpdate(13, 5, AddRef)}, + } + + messages, diag, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(diag.CoreSkipped) != 0 || diag.UndecodableBody != 0 || diag.UnknownPersister != 0 { + t.Fatalf("diag = %+v, want zero (core is now decoded, not skipped)", diag) + } + if len(messages) != 1 { + t.Fatalf("want 1 message, got %d", len(messages)) + } + m := messages[0] + if m.Core == nil { + t.Fatalf("Core = nil, want a decoded payload") + } + if m.Core.MessageID != 28 { + t.Errorf("Core.MessageID = %d, want 28", m.Core.MessageID) + } + if m.Core.Address != "salvage.core" { + t.Errorf("Core.Address = %q, want salvage.core", m.Core.Address) + } + if m.Core.Type != CoreTypeBytes { + t.Errorf("Core.Type = %d, want %d (BYTES)", m.Core.Type, CoreTypeBytes) + } + if !m.Core.Durable { + t.Errorf("Core.Durable = false, want true") + } + if len(m.Core.Body) != 120 { + t.Errorf("len(Core.Body) = %d, want 120", len(m.Core.Body)) + } + for i, b := range m.Core.Body { + if b != '.' { + t.Fatalf("Core.Body[%d] = %#x, want '.'", i, b) + } + } + if _, ok := m.Core.Properties["__AMQ_CID"]; !ok { + t.Errorf("Core.Properties missing __AMQ_CID; got keys %v", keysOf(m.Core.Properties)) + } + if len(m.QueueIDs) != 2 || m.QueueIDs[0] != 4 || m.QueueIDs[1] != 5 { + t.Errorf("QueueIDs = %v, want [4 5]", m.QueueIDs) + } +} + +func keysOf(m map[string]any) []string { + ks := make([]string, 0, len(m)) + for k := range m { + ks = append(ks, k) + } + return ks +} + +// TestDecodeCorePayloadRoundTrip covers the store (de)serialization of a +// decoded Core payload used by the Core store record. +func TestDecodeCorePayloadRoundTrip(t *testing.T) { + sv := Survivor{ID: 1, UserType: AddMessageProtocol, Body: loadCoreRecord(t), Updates: []RawRecord{buildRefUpdate(1, 1, AddRef)}} + messages, _, err := DecodeMessages([]Survivor{sv}) + if err != nil || len(messages) != 1 || messages[0].Core == nil { + t.Fatalf("decode: err=%v messages=%d", err, len(messages)) + } + orig := messages[0].Core + got, err := DecodeCorePayload(orig.Encode()) + if err != nil { + t.Fatalf("DecodeCorePayload: %v", err) + } + if got.MessageID != orig.MessageID || got.Address != orig.Address || got.Type != orig.Type || + got.Durable != orig.Durable || len(got.Body) != len(orig.Body) || len(got.Properties) != len(orig.Properties) { + t.Errorf("round-trip mismatch:\n got=%+v\norig=%+v", got, orig) + } +} + +func TestDecodeMessagesAddMessageSkipped(t *testing.T) { + // Legacy core ADD_MESSAGE (userType 31): out of scope for body decode, + // but still reported per queue via its refs. + sv := Survivor{ + ID: 14, + UserType: AddMessage, + Body: []byte{0xDE, 0xAD, 0xBE, 0xEF}, // opaque core encoding, never parsed + Updates: []RawRecord{buildRefUpdate(14, 9, AddRef)}, + } + + messages, diag, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 0 { + t.Fatalf("want 0 messages, got %d: %+v", len(messages), messages) + } + got := diag.CoreSkipped[14] + if len(got) != 1 || got[0] != 9 { + t.Errorf("diag.CoreSkipped[14] = %v, want [9]", got) + } +} + +func TestDecodeMessagesAddLargeMessageSkipped(t *testing.T) { + // Core large message (userType 30): out of scope, same treatment as + // ADD_MESSAGE. + sv := Survivor{ + ID: 15, + UserType: AddLargeMessage, + Body: []byte{0x01, 0x02}, + Updates: []RawRecord{buildRefUpdate(15, 6, AddRef)}, + } + + messages, diag, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 0 { + t.Fatalf("want 0 messages, got %d: %+v", len(messages), messages) + } + got := diag.CoreSkipped[15] + if len(got) != 1 || got[0] != 6 { + t.Errorf("diag.CoreSkipped[15] = %v, want [6]", got) + } +} + +func TestDecodeMessagesUndecodableBodyTruncated(t *testing.T) { + // Persister id 5 is recognized, but nothing follows it: reading the + // fixed prefix runs off the end of the body. + sv := Survivor{ + ID: 16, + UserType: AddMessageProtocol, + Body: []byte{persisterAMQPMessageV3}, + Updates: []RawRecord{buildRefUpdate(16, 1, AddRef)}, + } + + messages, diag, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 0 { + t.Fatalf("want 0 messages, got %d: %+v", len(messages), messages) + } + if diag.UndecodableBody != 1 { + t.Errorf("diag.UndecodableBody = %d, want 1", diag.UndecodableBody) + } + if diag.UnknownPersister != 0 || len(diag.CoreSkipped) != 0 { + t.Errorf("diag = %+v, want only UndecodableBody set", diag) + } +} + +func TestDecodeMessagesUndecodableEmptyBody(t *testing.T) { + sv := Survivor{ID: 17, UserType: AddMessageProtocol, Body: nil} + + messages, diag, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 0 { + t.Fatalf("want 0 messages, got %d: %+v", len(messages), messages) + } + if diag.UndecodableBody != 1 { + t.Errorf("diag.UndecodableBody = %d, want 1", diag.UndecodableBody) + } +} + +func TestDecodeMessagesZeroSurvivingRefsDropped(t *testing.T) { + sv := Survivor{ + ID: 18, + UserType: AddMessageProtocol, + Body: buildAMQPStandardBody(persisterAMQPMessageV3, []byte("body")), + Updates: []RawRecord{ + buildRefUpdate(18, 20, AddRef), + buildRefUpdate(18, 20, AcknowledgeRef), + }, + } + + messages, diag, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 0 { + t.Fatalf("want 0 messages (fully acked, nothing to salvage), got %d: %+v", len(messages), messages) + } + if diag.UnknownPersister != 0 || diag.UndecodableBody != 0 || len(diag.CoreSkipped) != 0 { + t.Errorf("diag = %+v, want zero (a fully-acked message isn't a decode failure)", diag) + } +} + +func TestDecodeMessagesPartialAckKeepsOtherQueue(t *testing.T) { + sv := Survivor{ + ID: 19, + UserType: AddMessageProtocol, + Body: buildAMQPStandardBody(persisterAMQPMessageV3, []byte("body")), + Updates: []RawRecord{ + buildRefUpdate(19, 20, AddRef), + buildRefUpdate(19, 21, AddRef), + buildRefUpdate(19, 20, AcknowledgeRef), + }, + } + + messages, _, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 1 { + t.Fatalf("want 1 message, got %d", len(messages)) + } + if len(messages[0].QueueIDs) != 1 || messages[0].QueueIDs[0] != 21 { + t.Errorf("QueueIDs = %v, want [21]", messages[0].QueueIDs) + } +} + +func TestDecodeMessagesScheduledUpdateSetsScheduledMs(t *testing.T) { + sv := Survivor{ + ID: 20, + UserType: AddMessageProtocol, + Body: buildAMQPStandardBody(persisterAMQPMessageV3, []byte("body")), + Updates: []RawRecord{ + buildRefUpdate(20, 30, AddRef), + buildScheduledUpdate(20, 30, 4102444800000), + }, + } + + messages, _, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 1 { + t.Fatalf("want 1 message, got %d", len(messages)) + } + if messages[0].ScheduledMs != 4102444800000 { + t.Errorf("ScheduledMs = %d, want 4102444800000", messages[0].ScheduledMs) + } +} + +func TestDecodeMessagesIgnoresUnrelatedUpdateTypes(t *testing.T) { + sv := Survivor{ + ID: 21, + UserType: AddMessageProtocol, + Body: buildAMQPStandardBody(persisterAMQPMessageV3, []byte("body")), + Updates: []RawRecord{ + buildRefUpdate(21, 40, AddRef), + {Type: UpdateRecord, ID: 21, UserType: UpdateDeliveryCount, Body: []byte{0, 0, 0, 3}}, + {Type: UpdateRecord, ID: 21, UserType: DuplicateID, Body: []byte("whatever")}, + }, + } + + messages, _, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 1 { + t.Fatalf("want 1 message, got %d", len(messages)) + } + if len(messages[0].QueueIDs) != 1 || messages[0].QueueIDs[0] != 40 { + t.Errorf("QueueIDs = %v, want [40]", messages[0].QueueIDs) + } + if messages[0].ScheduledMs != 0 { + t.Errorf("ScheduledMs = %d, want 0", messages[0].ScheduledMs) + } +} + +func TestDecodeMessagesMalformedRefBodySkipped(t *testing.T) { + sv := Survivor{ + ID: 22, + UserType: AddMessageProtocol, + Body: buildAMQPStandardBody(persisterAMQPMessageV3, []byte("body")), + Updates: []RawRecord{ + {Type: UpdateRecord, ID: 22, UserType: AddRef, Body: []byte{1, 2}}, // too short for an i64 queueID + }, + } + + messages, diag, err := DecodeMessages([]Survivor{sv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 0 { + t.Fatalf("want 0 messages (malformed ref => no surviving queue), got %d: %+v", len(messages), messages) + } + if diag.UnknownPersister != 0 || diag.UndecodableBody != 0 || len(diag.CoreSkipped) != 0 { + t.Errorf("diag = %+v, want zero (a malformed ref is skipped, not a body decode failure)", diag) + } +} + +func TestDecodeMessagesIgnoresPageCursorSurvivor(t *testing.T) { + messageSv := Survivor{ + ID: 23, + UserType: AddMessageProtocol, + Body: buildAMQPStandardBody(persisterAMQPMessageV3, []byte("body")), + Updates: []RawRecord{buildRefUpdate(23, 1, AddRef)}, + } + cursorSv := Survivor{ID: 24, UserType: PageCursorComplete, Body: []byte{1, 2, 3}} + txSv := Survivor{ID: 25, UserType: PageTransaction, Body: []byte{4, 5, 6}} + + messages, diag, err := DecodeMessages([]Survivor{messageSv, cursorSv, txSv}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 1 || messages[0].ID != 23 { + t.Fatalf("want exactly message 23, got %+v", messages) + } + if diag.UnknownPersister != 0 || diag.UndecodableBody != 0 || len(diag.CoreSkipped) != 0 { + t.Errorf("diag = %+v, want zero (page-cursor records are silently out of scope)", diag) + } +} + +func TestDecodeMessagesOutputSortedByAscendingID(t *testing.T) { + mk := func(id int64) Survivor { + return Survivor{ + ID: id, + UserType: AddMessageProtocol, + Body: buildAMQPStandardBody(persisterAMQPMessageV3, []byte("body")), + Updates: []RawRecord{buildRefUpdate(id, 1, AddRef)}, + } + } + + messages, _, err := DecodeMessages([]Survivor{mk(5), mk(1), mk(3)}) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + if len(messages) != 3 { + t.Fatalf("want 3 messages, got %d", len(messages)) + } + gotIDs := []int64{messages[0].ID, messages[1].ID, messages[2].ID} + want := []int64{1, 3, 5} + for i, id := range gotIDs { + if id != want[i] { + t.Errorf("messages[%d].ID = %d, want %d (order = %v)", i, id, want[i], gotIDs) + } + } +} + +// --- fixture tests: full pipe (ReadJournalDir -> Replayer -> DecodeMessages) +// against the harvested 2.42 broker data dir, cross-checked against +// manifest.json. --- + +// manifestEntry / manifestFile mirror testdata/manifest.json's schema +// (harvest_integration_test.go's manifestEntry, in the journal_test build +// tag reserved for the fixture generator). Duplicated here rather than +// shared since that file is gated behind ARTEMISCTL_HARVEST and lives in +// package journal_test, not journal. +type manifestEntry struct { + BodySha256 string `json:"bodySha256"` + BodyLen int `json:"bodyLen"` + Props map[string]any `json:"props,omitempty"` + ScheduledAtMs int64 `json:"scheduledAtMs,omitempty"` +} + +type manifestFile struct { + Queues map[string][]manifestEntry `json:"queues"` +} + +func loadManifest(t *testing.T) manifestFile { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "manifest.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var m manifestFile + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("unmarshal manifest: %v", err) + } + return m +} + +// decodeFixtureMessages runs the full pipe over the harvested fixture and +// returns the decoded messages, diag, and the queueID -> name map from the +// bindings journal (Task 6). +func decodeFixtureMessages(t *testing.T) ([]Message, MessageDiag, map[int64]string) { + t.Helper() + dir := fixtureDir(t) + + names, bdiags, err := ReadQueueBindings(filepath.Join(dir, "bindings")) + if err != nil { + t.Fatalf("ReadQueueBindings: %v", err) + } + if len(bdiags) != 0 { + t.Fatalf("bindings diags: %+v", bdiags) + } + + p := NewReplayer() + jdiags, err := ReadJournalDir(filepath.Join(dir, "journal"), "activemq-data", "amq", p.Feed) + if err != nil { + t.Fatalf("ReadJournalDir: %v", err) + } + if len(jdiags) != 0 { + t.Fatalf("journal diags: %+v", jdiags) + } + survivors, _ := p.Resolve() + + messages, diag, err := DecodeMessages(survivors) + if err != nil { + t.Fatalf("DecodeMessages: %v", err) + } + return messages, diag, names +} + +// byQueue groups decoded messages by their single queue name. The fixture +// sends every message point-to-point to exactly one queue, so every decoded +// message is expected to carry exactly one surviving queueID. +func byQueue(t *testing.T, messages []Message, names map[int64]string) map[string][]Message { + t.Helper() + out := make(map[string][]Message) + for _, m := range messages { + if len(m.QueueIDs) != 1 { + t.Fatalf("message %d has %d queueIDs, want 1: %v", m.ID, len(m.QueueIDs), m.QueueIDs) + } + name, ok := names[m.QueueIDs[0]] + if !ok { + t.Fatalf("message %d: queueID %d has no binding", m.ID, m.QueueIDs[0]) + } + out[name] = append(out[name], m) + } + return out +} + +func sha256Hex(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +// verifyQueueBodies asserts got and want are the same set of messages +// (by body sha256/len), regardless of order. Set membership (not index +// alignment) because the manifest's per-message order has no relationship to +// the journal record IDs assigned at send time. +func verifyQueueBodies(t *testing.T, got []Message, want []manifestEntry, requireExactCount bool) { + t.Helper() + if requireExactCount && len(got) != len(want) { + t.Fatalf("got %d messages, want %d", len(got), len(want)) + } + wantLenByHash := make(map[string]int, len(want)) + for _, e := range want { + wantLenByHash[e.BodySha256] = e.BodyLen + } + seen := make(map[string]bool, len(got)) + for _, m := range got { + var am amqp.Message + if err := am.UnmarshalBinary(m.AMQP); err != nil { + t.Fatalf("unmarshal AMQP for message %d: %v", m.ID, err) + } + body := am.GetData() + hash := sha256Hex(body) + wantLen, ok := wantLenByHash[hash] + if !ok { + t.Fatalf("message %d body sha256 %s not present in manifest", m.ID, hash) + } + if wantLen != len(body) { + t.Errorf("message %d body len = %d, want %d", m.ID, len(body), wantLen) + } + if seen[hash] { + t.Errorf("message %d body sha256 %s seen more than once", m.ID, hash) + } + seen[hash] = true + } +} + +func TestDecodeMessagesFixturePlain(t *testing.T) { + messages, _, names := decodeFixtureMessages(t) + man := loadManifest(t) + byQ := byQueue(t, messages, names) + + verifyQueueBodies(t, byQ["salvage.plain"], man.Queues["salvage.plain"], true) +} + +func TestDecodeMessagesFixtureProps(t *testing.T) { + messages, _, names := decodeFixtureMessages(t) + man := loadManifest(t) + byQ := byQueue(t, messages, names) + + got := byQ["salvage.props"] + want := man.Queues["salvage.props"] + verifyQueueBodies(t, got, want, true) + + for _, m := range got { + var am amqp.Message + if err := am.UnmarshalBinary(m.AMQP); err != nil { + t.Fatalf("unmarshal AMQP for message %d: %v", m.ID, err) + } + region, ok := am.ApplicationProperties["region"] + if !ok || fmt.Sprint(region) != "eu" { + t.Errorf("message %d ApplicationProperties[region] = %v, want eu", m.ID, region) + } + attempt, ok := am.ApplicationProperties["attempt"] + if !ok || fmt.Sprint(attempt) != "1" { + t.Errorf("message %d ApplicationProperties[attempt] = %v, want 1", m.ID, attempt) + } + } +} + +func TestDecodeMessagesFixtureScheduled(t *testing.T) { + messages, _, names := decodeFixtureMessages(t) + man := loadManifest(t) + byQ := byQueue(t, messages, names) + + got := byQ["salvage.scheduled"] + want := man.Queues["salvage.scheduled"] + verifyQueueBodies(t, got, want, true) + + if len(got) != 1 { + t.Fatalf("want 1 scheduled message, got %d", len(got)) + } + if len(want) != 1 { + t.Fatalf("manifest: want 1 scheduled entry, got %d", len(want)) + } + if got[0].ScheduledMs != want[0].ScheduledAtMs { + t.Errorf("ScheduledMs = %d, want %d", got[0].ScheduledMs, want[0].ScheduledAtMs) + } +} + +func TestDecodeMessagesFixtureLarge(t *testing.T) { + messages, _, names := decodeFixtureMessages(t) + byQ := byQueue(t, messages, names) + + got := byQ["salvage.large"] + if len(got) != 1 { + t.Fatalf("want 1 large message, got %d: %+v", len(got), got) + } + if !got[0].Large { + t.Errorf("Large = false, want true") + } +} + +func TestDecodeMessagesFixtureAcked(t *testing.T) { + messages, _, names := decodeFixtureMessages(t) + byQ := byQueue(t, messages, names) + + if got := byQ["salvage.acked"]; len(got) != 0 { + t.Fatalf("want 0 acked messages, got %d: %+v", len(got), got) + } +} + +// TestDecodeMessagesFixturePagedSpillover covers format_notes.md section 8's +// "paging is a spillover, not a mirror" finding: 44 of the 500 +// salvage.paged messages were sent before the address crossed +// maxSizeBytes and landed in the journal like normal messages (the other +// 456 live in page files, out of scope for this reader -- Task 9). The +// brief's original "assert salvage.paged yields 0 here" is wrong for this +// fixture; the corrected count comes from format_notes.md's fixture census. +func TestDecodeMessagesFixturePagedSpillover(t *testing.T) { + messages, _, names := decodeFixtureMessages(t) + man := loadManifest(t) + byQ := byQueue(t, messages, names) + + got := byQ["salvage.paged"] + if len(got) != 44 { + t.Fatalf("want 44 journal-resident salvage.paged messages, got %d", len(got)) + } + // Subset check only (requireExactCount=false): got is 44 of the + // manifest's full 500-entry set, not all of it. + verifyQueueBodies(t, got, man.Queues["salvage.paged"], false) +} + +// TestDecodeMessagesFixtureDiagClean asserts the fixture -- entirely +// AMQP-produced, no Core-protocol messages -- decodes with zero skips. +func TestDecodeMessagesFixtureDiagClean(t *testing.T) { + _, diag, _ := decodeFixtureMessages(t) + + if diag.UnknownPersister != 0 { + t.Errorf("diag.UnknownPersister = %d, want 0", diag.UnknownPersister) + } + if diag.UndecodableBody != 0 { + t.Errorf("diag.UndecodableBody = %d, want 0", diag.UndecodableBody) + } + if len(diag.CoreSkipped) != 0 { + t.Errorf("diag.CoreSkipped = %v, want empty", diag.CoreSkipped) + } +} + +// TestDecodeMessagesFixtureTotalCount cross-checks against +// TestReplayFixtureMessageJournal's independently-derived "56 +// AddMessageProtocol survivors" figure (5 plain + 5 props + 1 scheduled + 1 +// large + 44 paged = 56). +func TestDecodeMessagesFixtureTotalCount(t *testing.T) { + messages, _, _ := decodeFixtureMessages(t) + if len(messages) != 56 { + t.Fatalf("want 56 decoded messages, got %d", len(messages)) + } +} diff --git a/internal/journal/paging.go b/internal/journal/paging.go new file mode 100644 index 0000000..bc40d1b --- /dev/null +++ b/internal/journal/paging.go @@ -0,0 +1,473 @@ +package journal + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" +) + +// pageStartByte / pageEndByte: page-file entry framing (format_notes.md +// section 8, PageReadWriter's START_BYTE/END_BYTE). +const ( + pageStartByte byte = '{' + pageEndByte byte = '}' +) + +// pageLargeMessageType constants (format_notes.md section 8's "byte +// largeMessageType" field). Only NONE/NOT_CORE take the +// MessagePersister.decode path this AMQP-only reader understands; +// CORE/OLD_CORE carry a core-large-persister header it cannot decode. +const ( + pageLargeTypeOldCore int8 = -1 + pageLargeTypeNone int8 = 0 + pageLargeTypeCore int8 = 1 + pageLargeTypeNotCore int8 = 2 +) + +// pageFilePattern matches Artemis page file names (format_notes.md section +// 8, fixture-verified: "000000001.page", "000000002.page", ...). +var pageFilePattern = regexp.MustCompile(`^(\d+)\.page$`) + +// CursorState is the best-effort page-cursor knowledge extracted from the +// message journal's ACKNOWLEDGE_CURSOR (39) / PAGE_CURSOR_COMPLETE (42) +// survivors (format_notes.md section 10, added by this task -- deferred by +// Task 7, see message.go's DecodeMessages design note). Both record +// families share one on-disk encoding keyed by queueID + page position. +type CursorState struct { + // CompletePages[queueID][pageNr] marks a page fully consumed by queueID + // (PAGE_CURSOR_COMPLETE): every entry in that page can be skipped for + // that queue without inspecting individual acks. + CompletePages map[int64]map[int64]bool + // AckedEntries[queueID][{pageNr, messageNr}] marks one page entry -- + // identified by its 0-based ordinal position within its page file, see + // format_notes.md section 10's Page.java citation -- individually acked + // by queueID (ACKNOWLEDGE_CURSOR). + AckedEntries map[int64]map[[2]int64]bool +} + +// BuildCursorState scans message-journal survivors (the same []Survivor +// Replayer.Resolve returns, also consumed by DecodeMessages) for the +// page-cursor record family DecodeMessages's design note explicitly leaves +// for this task, and folds them into a CursorState for ReadPaging's +// best-effort filtering. Survivors of any other UserType are ignored. +// +// The returned CursorState's maps are always non-nil, even for empty input, +// so callers can index them directly without a nil check. +func BuildCursorState(survivors []Survivor) CursorState { + cs := CursorState{ + CompletePages: make(map[int64]map[int64]bool), + AckedEntries: make(map[int64]map[[2]int64]bool), + } + + for _, sv := range survivors { + switch sv.UserType { + case AcknowledgeCursor: + queueID, pageNr, messageNr, ok := decodeCursorAck(sv.Body) + if !ok { + continue + } + if cs.AckedEntries[queueID] == nil { + cs.AckedEntries[queueID] = make(map[[2]int64]bool) + } + cs.AckedEntries[queueID][[2]int64{pageNr, int64(messageNr)}] = true + + case PageCursorComplete: + queueID, pageNr, _, ok := decodeCursorAck(sv.Body) + if !ok { + continue + } + if cs.CompletePages[queueID] == nil { + cs.CompletePages[queueID] = make(map[int64]bool) + } + cs.CompletePages[queueID][pageNr] = true + + default: + // Every other survivor family (messages, PAGE_TRANSACTION, + // PAGE_CURSOR_COUNTER_VALUE/INC, ...) is out of scope here, same + // as DecodeMessages's default case. + } + } + return cs +} + +// decodeCursorAck decodes a CursorAckRecordEncoding body (format_notes.md +// section 10): long queueID, long pageNr, int messageNr. Shared by both +// ACKNOWLEDGE_CURSOR and PAGE_CURSOR_COMPLETE, which use the identical +// on-disk layout. +func decodeCursorAck(body []byte) (queueID, pageNr int64, messageNr int32, ok bool) { + r := newReader(body) + queueID = r.i64() + pageNr = r.i64() + messageNr = r.i32() + if r.err() != nil { + return 0, 0, 0, false + } + return queueID, pageNr, messageNr, true +} + +// PagedMessage is one message recovered from a page file. +type PagedMessage struct { + AMQP []byte + Core *CorePayload // non-nil for a decoded Core-protocol paged entry (AMQP empty) + QueueIDs []int64 + // ScheduledMs is always 0: format_notes.md section 8's PagedMessage + // field table has no scheduled-delivery field on disk (unlike + // message.go's Message, which gets it from a separate + // SET_SCHEDULED_DELIVERY_TIME journal update). Kept for field-shape + // symmetry with Message per the task-9 brief's struct definition. + ScheduledMs int64 +} + +// PagingDiag: skips + best-effort notes. +// +// Fields beyond the task-9 brief's three (LargeSkipped, UnknownPersister, +// UndecodableEntries) are an intentional deviation, documented in the task +// report: they mirror message.go's MessageDiag granularity (which itself +// went beyond its own brief) rather than folding every non-exportable entry +// into CoreSkipped, which would misreport why an entry was skipped. +type PagingDiag struct { + CoreSkipped int // persister-id Core (1), or largeMessageType CORE/OLD_CORE entries this reader cannot decode + CorruptPages []FileDiag // page files where framing broke down (bad START/END byte, or a size that overruns EOF); entries decoded before the break are still kept + PagesSkippedComplete int // whole pages skipped via PAGE_CURSOR_COMPLETE (every queue the page's entries target has it marked complete) + + LargeSkipped int // paged AMQP-large entries (persister id 4): body lives outside the page file, out of scope for this reader (format_notes.md sections 5b/8; correction #4 -- none observed in the fixture, handled defensively) + UnknownPersister int // entries whose persister-id byte isn't recognized + UndecodableEntries int // well-framed entries whose PagedMessage payload couldn't be parsed past a recognized point + + DirMissing bool // pagingDir did not exist; distinguish from an existing-but-empty dir +} + +// ReadPaging walks //*.page in page-number order, +// decodes size-prefixed PagedMessage entries ('{' size bytes '}'), and +// filters best-effort against cursor state (format_notes.md sections 8/10): +// a page marked complete for every queue any of its exportable entries +// target is skipped as a whole (PagesSkippedComplete); within other pages, +// an entry individually acked (ACKNOWLEDGE_CURSOR) for every queue it +// targets is skipped; otherwise it is exported (at-least-once, spec §4). +// +// A missing pagingDir is not an error: it means nothing was ever paged. +func ReadPaging(pagingDir string, cursors CursorState) ([]PagedMessage, PagingDiag, error) { + var diag PagingDiag + var out []PagedMessage + + addrEntries, err := os.ReadDir(pagingDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + diag.DirMissing = true + return nil, diag, nil + } + return nil, diag, fmt.Errorf("journal: read paging dir %s: %w", pagingDir, err) + } + + var addrDirs []string + for _, e := range addrEntries { + if e.IsDir() { + addrDirs = append(addrDirs, e.Name()) + } + } + // Deterministic order across address dirs. address.txt (the address-name + // marker file, format_notes.md section 8) is intentionally never read: + // neither PagedMessage nor PagingDiag surfaces an address, since + // queueIDs -- which do ride in each entry -- are all this reader needs. + sort.Strings(addrDirs) + + for _, addrName := range addrDirs { + addrDir := filepath.Join(pagingDir, addrName) + pageFiles, err := listPageFiles(addrDir) + if err != nil { + return nil, diag, err + } + + for _, pf := range pageFiles { + path := filepath.Join(addrDir, pf.name) + data, err := os.ReadFile(path) //nolint:gosec // salvage reads operator-supplied broker data dirs by design + if err != nil { + return nil, diag, fmt.Errorf("journal: read page file %s: %w", path, err) + } + readOnePageFile(path, data, pf.pageNr, cursors, &out, &diag) + } + } + + return out, diag, nil +} + +// PagingReferenced reports whether any message-journal survivor is evidence of +// paging activity: PAGE_TRANSACTION (35), ACKNOWLEDGE_CURSOR (39), +// PAGE_CURSOR_COUNTER_VALUE (40), PAGE_CURSOR_COUNTER_INC (41), +// PAGE_CURSOR_COMPLETE (42), PAGE_PENDING_COUNTER (43). The orchestrator uses +// it to warn when the journal references paging but the paging dir is missing. +func PagingReferenced(survivors []Survivor) bool { + for _, sv := range survivors { + switch sv.UserType { + case PageTransaction, AcknowledgeCursor, PageCursorCounterValue, PageCursorCounterInc, PageCursorComplete, PageCursorPendingCounter: + return true + } + } + return false +} + +// pageFileInfo is one page file's name plus its parsed page number (the +// numeric prefix before ".page", which doubles as the pageNr half of a +// cursor position, format_notes.md section 10). +type pageFileInfo struct { + name string + pageNr int64 +} + +// listPageFiles returns dir's *.page entries in ascending page-number order. +// Non-.page files (notably address.txt) are silently skipped. +func listPageFiles(dir string) ([]pageFileInfo, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("journal: read paging address dir %s: %w", dir, err) + } + + var out []pageFileInfo + for _, e := range entries { + if e.IsDir() { + continue + } + m := pageFilePattern.FindStringSubmatch(e.Name()) + if m == nil { + continue + } + n, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + // Unreachable: the regex already guarantees an all-digit group. + continue + } + out = append(out, pageFileInfo{name: e.Name(), pageNr: n}) + } + sort.Slice(out, func(i, j int) bool { return out[i].pageNr < out[j].pageNr }) + return out, nil +} + +// pagedCandidate is one exportable (persisterDecodeStandard) entry decoded +// from a page file, pending cursor filtering. +type pagedCandidate struct { + amqp []byte + core *CorePayload + queueIDs []int64 + messageNr int64 +} + +// readOnePageFile decodes one page file's entries and applies best-effort +// cursor filtering, appending survivors to *out and updating *diag in place. +func readOnePageFile(path string, data []byte, pageNr int64, cursors CursorState, out *[]PagedMessage, diag *PagingDiag) { + frames, badOffset, corrupt := readPageFileFrames(data) + if corrupt { + diag.CorruptPages = append(diag.CorruptPages, FileDiag{ + Path: path, + Offset: badOffset, + Reason: "bad page-entry framing", + Corrupt: true, // structural page-file damage: gates salvage's exit code (spec §1/§5) + }) + } + + var candidates []pagedCandidate + queueSet := make(map[int64]bool) + + for i, frame := range frames { + amqpBytes, core, queueIDs, outcome := decodePagedMessage(frame) + switch outcome { + case persisterDecodeCore: + if core == nil { + // Core large paged entry (body outside the page file): cannot export. + diag.CoreSkipped++ + break + } + for _, q := range queueIDs { + queueSet[q] = true + } + candidates = append(candidates, pagedCandidate{core: core, queueIDs: queueIDs, messageNr: int64(i)}) + case persisterDecodeLarge: + diag.LargeSkipped++ + case persisterDecodeUnknown: + diag.UnknownPersister++ + case persisterDecodeMalformed: + diag.UndecodableEntries++ + case persisterDecodeStandard: + for _, q := range queueIDs { + queueSet[q] = true + } + candidates = append(candidates, pagedCandidate{amqp: amqpBytes, queueIDs: queueIDs, messageNr: int64(i)}) + } + } + + if len(candidates) == 0 { + return + } + + if pageComplete(queueSet, pageNr, cursors) { + diag.PagesSkippedComplete++ + return + } + + for _, c := range candidates { + if entryAcked(c.queueIDs, pageNr, c.messageNr, cursors) { + continue + } + *out = append(*out, PagedMessage{AMQP: c.amqp, Core: c.core, QueueIDs: c.queueIDs}) + } +} + +// pageComplete reports whether every queue any candidate entry in this page +// targets has that page marked complete (CursorState.CompletePages, +// format_notes.md section 10's PAGE_CURSOR_COMPLETE). An empty queueSet +// (defensive; every real entry carries at least one queueID) is never +// "complete" -- there is nothing to have completed. +func pageComplete(queueSet map[int64]bool, pageNr int64, cursors CursorState) bool { + if len(queueSet) == 0 { + return false + } + for q := range queueSet { + pages := cursors.CompletePages[q] + if pages == nil || !pages[pageNr] { + return false + } + } + return true +} + +// entryAcked reports whether every queue this entry targets has +// individually acked this exact position (CursorState.AckedEntries, +// format_notes.md section 10's ACKNOWLEDGE_CURSOR). +func entryAcked(queueIDs []int64, pageNr, messageNr int64, cursors CursorState) bool { + if len(queueIDs) == 0 { + return false + } + pos := [2]int64{pageNr, messageNr} + for _, q := range queueIDs { + acked := cursors.AckedEntries[q] + if acked == nil || !acked[pos] { + return false + } + } + return true +} + +// readPageFileFrames scans one page file's outer '{' size '}' framing +// (format_notes.md section 8), returning every well-framed entry body in +// on-disk order. It stops at the first framing violation (bad start/end +// byte, or a size that would run past EOF) rather than resyncing +// byte-by-byte like the message/bindings journal reader (file.go): +// format_notes.md section 8 notes a bad start/end byte "marks the file +// suspect (partial trailing write tolerated)" -- i.e. real Artemis's own +// page reader treats this as the normal end of valid data (a page file +// actively being appended to when the broker died), not corruption to +// recover past. Whatever decoded cleanly before the violation is kept, and +// the caller records one diagnostic for the incident. +func readPageFileFrames(data []byte) (frames [][]byte, badOffset int64, corrupt bool) { + pos := 0 + for pos < len(data) { + if data[pos] != pageStartByte { + return frames, int64(pos), true + } + if pos+5 > len(data) { + return frames, int64(pos), true + } + r := newReader(data[pos+1 : pos+5]) + size := int(r.i32()) + start := pos + 5 + end := start + size + if size < 0 || end < start || end+1 > len(data) { + return frames, int64(pos), true + } + if data[end] != pageEndByte { + return frames, int64(pos), true + } + frames = append(frames, data[start:end]) + pos = end + 1 + } + return frames, 0, false +} + +// skipPersisterTail consumes the V2/V3 tail decodePersisterPayload leaves +// unread (format_notes.md section 5a): int extraPropsSize + that many +// bytes (V2 id=3, V3 id=5 only -- V1 id=2 has neither field), then a +// V3-only 8-byte expiration long. paging.go needs this to reach the +// queueIDsCount field that follows a PagedMessage's persister payload +// (format_notes.md section 8); message.go never calls this since nothing +// follows an ADD_MESSAGE_PROTOCOL record's persister payload. +func skipPersisterTail(r *reader, persisterID byte) { + if persisterID != persisterAMQPMessageV2 && persisterID != persisterAMQPMessageV3 { + return + } + extraPropsSize := r.i32() + if r.err() != nil { + return + } + if extraPropsSize > 0 { + r.bytes(int(extraPropsSize)) + } + if persisterID == persisterAMQPMessageV3 { + r.i64() + } +} + +// decodePagedMessage decodes one page-file entry's PagedMessage payload +// (format_notes.md section 8) far enough to export it: transactionID +// (read and discarded, see below), largeMessageType, the AMQP-standard +// persister payload (shared with message.go via decodePersisterPayload), +// its V2/V3 tail (needed to reach queueIDsCount), then queueIDsCount + +// queueIDs. +// +// Entries this v1 reader cannot or does not export (CORE/OLD_CORE large +// types, or a Core/AMQP-large/unknown/malformed persister) return early +// with just the outcome the caller uses to pick a diagnostic counter -- +// their queueIDs are never needed, since a skipped entry is never +// individually cursor-filtered and the entry's byte range is already known +// from the outer '{' size '}' framing (readPageFileFrames), so skipping it +// does not require parsing all the way through. +// +// The entry's own transactionID (format_notes.md section 8: the fixture's +// only observed value is -1, non-transactional) is read and discarded: +// ReadPaging has no paging-transaction log to cross-reference it against +// (PAGE_TRANSACTION, userType 35, is out of this reader's scope per +// message.go's DecodeMessages design note), so every entry is treated as +// committed and exported at-least-once (spec §4) regardless. +func decodePagedMessage(body []byte) (amqpBytes []byte, core *CorePayload, queueIDs []int64, outcome persisterDecodeOutcome) { + r := newReader(body) + r.i64() // transactionID + largeType := int8(r.u8()) + if r.err() != nil { + return nil, nil, nil, persisterDecodeMalformed + } + + if largeType != pageLargeTypeNone && largeType != pageLargeTypeNotCore { + // CORE or OLD_CORE large: the body lives outside the page file + // (large-messages dir), so a paged core-large entry cannot be exported + // from the page file alone -- reported as CoreSkipped (core == nil). + return nil, nil, nil, persisterDecodeCore + } + + amqp, corePayload, persisterID, pOutcome := decodePersisterPayload(r) + switch pOutcome { + case persisterDecodeStandard: + skipPersisterTail(r, persisterID) + case persisterDecodeCore: + // Core payload is fully self-delimiting (decodeCoreStandardBody + // consumed body+headers+properties); no V2/V3 tail follows. + default: + return nil, nil, nil, pOutcome + } + + qCount := r.i32() + if r.err() != nil || qCount < 0 || qCount > int32(r.remaining()/8) { + return nil, nil, nil, persisterDecodeMalformed + } + ids := make([]int64, 0, qCount) + for i := int32(0); i < qCount; i++ { + ids = append(ids, r.i64()) + } + if r.err() != nil { + return nil, nil, nil, persisterDecodeMalformed + } + + return amqp, corePayload, ids, pOutcome +} diff --git a/internal/journal/paging_test.go b/internal/journal/paging_test.go new file mode 100644 index 0000000..714df95 --- /dev/null +++ b/internal/journal/paging_test.go @@ -0,0 +1,640 @@ +package journal + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Azure/go-amqp" +) + +// amqpDataSection unmarshals raw AMQP-encoded message bytes and returns the +// Data section's body, mirroring message_test.go's verifyQueueBodies. +func amqpDataSection(raw []byte) ([]byte, error) { + var am amqp.Message + if err := am.UnmarshalBinary(raw); err != nil { + return nil, err + } + return am.GetData(), nil +} + +// --- synthetic byte-builders (mirror format_notes.md sections 8/10's +// write-side layouts; the beI32/beI64/encodeNullableSimpleString helpers +// live in message_test.go, same package) --- + +// buildPagedMessagePayload builds one PagedMessage payload (format_notes.md +// section 8): long transactionID (-1, non-transactional -- the fixture's +// only observed value), byte largeMessageType (0 = NONE), then +// MessagePersister.decode = [persisterID][persister payload from section 5a], +// then int queueIDsCount + queueIDs. For V2/V3 persisters it also encodes an +// empty extraProps block (and, for V3, a zero expiration) so the entry's +// true on-disk length matches what a real broker would have written -- +// ReadPaging needs to walk past that tail to reach queueIDsCount. +func buildPagedMessagePayload(persisterID byte, amqpBytes []byte, queueIDs []int64) []byte { + body := beI64(-1) // transactionID + body = append(body, byte(pageLargeTypeNone)) // largeMessageType + body = append(body, persisterID) + body = append(body, beI64(77)...) // messageID (persister-internal, unused by the reader) + body = append(body, beI64(0)...) // messageFormat + body = append(body, encodeNullableSimpleString("test.paged", true)...) + body = append(body, beI32(int32(len(amqpBytes)))...) + body = append(body, amqpBytes...) + if persisterID == persisterAMQPMessageV2 || persisterID == persisterAMQPMessageV3 { + body = append(body, beI32(0)...) // extraPropsSize = 0 (none) + if persisterID == persisterAMQPMessageV3 { + body = append(body, beI64(0)...) // expiration + } + } + body = append(body, beI32(int32(len(queueIDs)))...) + for _, q := range queueIDs { + body = append(body, beI64(q)...) + } + return body +} + +// encodePageEntry wraps a PagedMessage payload in the page-file entry +// framing (format_notes.md section 8): START_BYTE '{', int size, payload, +// END_BYTE '}'. +func encodePageEntry(payload []byte) []byte { + out := []byte{pageStartByte} + out = append(out, beI32(int32(len(payload)))...) + out = append(out, payload...) + out = append(out, pageEndByte) + return out +} + +// encodeCursorAckBody builds a CursorAckRecordEncoding body (format_notes.md +// section 10, shared by ACKNOWLEDGE_CURSOR and PAGE_CURSOR_COMPLETE): long +// queueID, long pageNr, int messageNr. +func encodeCursorAckBody(queueID, pageNr int64, messageNr int32) []byte { + b := beI64(queueID) + b = append(b, beI64(pageNr)...) + b = append(b, beI32(messageNr)...) + return b +} + +// writePageFile writes data as //, creating +// directories as needed, and returns the paging root dir (the ReadPaging +// argument). +func writePageFile(t *testing.T, addrDirName, pageFileName string, data []byte) string { + t.Helper() + root := t.TempDir() + addrDir := filepath.Join(root, addrDirName) + if err := os.MkdirAll(addrDir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", addrDir, err) + } + if err := os.WriteFile(filepath.Join(addrDir, pageFileName), data, 0o644); err != nil { + t.Fatalf("write %s: %v", pageFileName, err) + } + return root +} + +// --- BuildCursorState --- + +func TestBuildCursorStateDecodesAckAndComplete(t *testing.T) { + survivors := []Survivor{ + {ID: 100, UserType: AcknowledgeCursor, Body: encodeCursorAckBody(5, 2, 7)}, + {ID: 101, UserType: PageCursorComplete, Body: encodeCursorAckBody(5, 3, 0)}, + // Unrelated survivor families must not be touched. + {ID: 102, UserType: AddMessageProtocol, Body: []byte{persisterAMQPMessageV3}}, + {ID: 103, UserType: PageCursorCounterValue, Body: []byte{1, 2, 3, 4}}, + } + + cs := BuildCursorState(survivors) + + if !cs.AckedEntries[5][[2]int64{2, 7}] { + t.Errorf("AckedEntries[5][{2,7}] = false, want true") + } + if len(cs.AckedEntries[5]) != 1 { + t.Errorf("AckedEntries[5] has %d entries, want 1: %v", len(cs.AckedEntries[5]), cs.AckedEntries[5]) + } + if !cs.CompletePages[5][3] { + t.Errorf("CompletePages[5][3] = false, want true") + } + if len(cs.CompletePages[5]) != 1 { + t.Errorf("CompletePages[5] has %d entries, want 1", len(cs.CompletePages[5])) + } +} + +func TestBuildCursorStateMalformedBodyIgnored(t *testing.T) { + survivors := []Survivor{ + {ID: 100, UserType: AcknowledgeCursor, Body: []byte{1, 2}}, // too short for the 20-byte encoding + } + cs := BuildCursorState(survivors) + if len(cs.AckedEntries) != 0 { + t.Errorf("AckedEntries = %v, want empty", cs.AckedEntries) + } +} + +func TestBuildCursorStateEmptyInput(t *testing.T) { + cs := BuildCursorState(nil) + if cs.CompletePages == nil || cs.AckedEntries == nil { + t.Fatalf("maps must be initialized (non-nil) even for empty input: %+v", cs) + } + if len(cs.CompletePages) != 0 || len(cs.AckedEntries) != 0 { + t.Errorf("want empty maps, got %+v", cs) + } +} + +// --- ReadPaging: synthetic cursor filtering (brief step 1b) --- + +func TestReadPagingCursorFilteringAckedEntrySkipped(t *testing.T) { + const queueID = int64(5) + const pageNr = int64(1) + + entries := [][]byte{ + buildPagedMessagePayload(persisterAMQPMessageV3, []byte("entry-zero-body"), []int64{queueID}), + buildPagedMessagePayload(persisterAMQPMessageV3, []byte("entry-one-body"), []int64{queueID}), + buildPagedMessagePayload(persisterAMQPMessageV3, []byte("entry-two-body"), []int64{queueID}), + } + var page []byte + for _, e := range entries { + page = append(page, encodePageEntry(e)...) + } + root := writePageFile(t, "addr1", "000000001.page", page) + + // Entry index 1 (0-based, per format_notes.md section 10's messageNr = + // ordinal position within the page) acked for its only queue. + cursors := CursorState{ + AckedEntries: map[int64]map[[2]int64]bool{ + queueID: {{pageNr, 1}: true}, + }, + } + + got, diag, err := ReadPaging(root, cursors) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(got) != 2 { + t.Fatalf("want 2 surviving entries, got %d: %+v", len(got), got) + } + if diag.PagesSkippedComplete != 0 { + t.Errorf("PagesSkippedComplete = %d, want 0", diag.PagesSkippedComplete) + } + bodies := map[string]bool{} + for _, m := range got { + bodies[string(m.AMQP)] = true + } + if !bodies["entry-zero-body"] || !bodies["entry-two-body"] { + t.Errorf("got bodies %v, want entry-zero-body and entry-two-body (entry-one-body acked)", bodies) + } + if bodies["entry-one-body"] { + t.Errorf("entry-one-body should have been filtered as acked") + } +} + +func TestReadPagingCursorFilteringPageCompleteSkipsAllEntries(t *testing.T) { + const queueID = int64(5) + const pageNr = int64(1) + + entries := [][]byte{ + buildPagedMessagePayload(persisterAMQPMessageV3, []byte("entry-a"), []int64{queueID}), + buildPagedMessagePayload(persisterAMQPMessageV3, []byte("entry-b"), []int64{queueID}), + buildPagedMessagePayload(persisterAMQPMessageV3, []byte("entry-c"), []int64{queueID}), + } + var page []byte + for _, e := range entries { + page = append(page, encodePageEntry(e)...) + } + root := writePageFile(t, "addr1", "000000001.page", page) + + cursors := CursorState{ + CompletePages: map[int64]map[int64]bool{ + queueID: {pageNr: true}, + }, + } + + got, diag, err := ReadPaging(root, cursors) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(got) != 0 { + t.Fatalf("want 0 surviving entries (page complete), got %d: %+v", len(got), got) + } + if diag.PagesSkippedComplete != 1 { + t.Errorf("PagesSkippedComplete = %d, want 1", diag.PagesSkippedComplete) + } +} + +func TestReadPagingPageCompletePartialQueuesNotSkipped(t *testing.T) { + // Two entries in the same page target different queues; only one + // queue's cursor marks the page complete. format_notes.md section 10: + // completion is per (queueID, pageNr) -- partial completion across the + // page's queues must not wholesale-skip the page (at-least-once, spec + // §4). + const pageNr = int64(1) + payloadA := buildPagedMessagePayload(persisterAMQPMessageV3, []byte("queueA-entry"), []int64{1}) + payloadB := buildPagedMessagePayload(persisterAMQPMessageV3, []byte("queueB-entry"), []int64{2}) + page := append(encodePageEntry(payloadA), encodePageEntry(payloadB)...) + root := writePageFile(t, "addr1", "000000001.page", page) + + cursors := CursorState{ + CompletePages: map[int64]map[int64]bool{ + 1: {pageNr: true}, // only queue 1's page marked complete + }, + } + + got, diag, err := ReadPaging(root, cursors) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if diag.PagesSkippedComplete != 0 { + t.Errorf("PagesSkippedComplete = %d, want 0 (only 1 of 2 queues complete)", diag.PagesSkippedComplete) + } + if len(got) != 2 { + t.Fatalf("want both entries exported, got %d: %+v", len(got), got) + } +} + +// --- ReadPaging: corrupt entry (brief step 1c) --- + +func TestReadPagingCorruptEntryKeepsEarlierEntries(t *testing.T) { + good := encodePageEntry(buildPagedMessagePayload(persisterAMQPMessageV3, []byte("good-entry-body"), []int64{1})) + bad := encodePageEntry(buildPagedMessagePayload(persisterAMQPMessageV3, []byte("second-entry-body"), []int64{1})) + bad[len(bad)-1] = 0x00 // corrupt the trailing END_BYTE + + page := append(append([]byte{}, good...), bad...) + root := writePageFile(t, "addr1", "000000001.page", page) + + got, diag, err := ReadPaging(root, CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(got) != 1 { + t.Fatalf("want 1 surviving entry (earlier one kept), got %d: %+v", len(got), got) + } + if string(got[0].AMQP) != "good-entry-body" { + t.Errorf("AMQP = %q, want good-entry-body", got[0].AMQP) + } + if len(diag.CorruptPages) != 1 { + t.Fatalf("want 1 CorruptPages diag, got %d: %+v", len(diag.CorruptPages), diag.CorruptPages) + } +} + +func TestReadPagingCorruptStartByteStopsFile(t *testing.T) { + good := encodePageEntry(buildPagedMessagePayload(persisterAMQPMessageV3, []byte("only-good-body"), []int64{1})) + page := append(append([]byte{}, good...), 0xFF) // trailing junk byte: not a valid START_BYTE + root := writePageFile(t, "addr1", "000000001.page", page) + + got, diag, err := ReadPaging(root, CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(got) != 1 || string(got[0].AMQP) != "only-good-body" { + t.Fatalf("want 1 entry only-good-body, got %+v", got) + } + if len(diag.CorruptPages) != 1 { + t.Errorf("want 1 CorruptPages diag, got %d", len(diag.CorruptPages)) + } +} + +// --- ReadPaging: Core-persisted entry (brief step 1d) --- + +func TestReadPagingCorePersisterExported(t *testing.T) { + // A well-formed persister-1 Core entry (the golden 2.42.0 payload) followed + // by a queueIDs list is now decoded and exported, not skipped. + golden, rerr := os.ReadFile(filepath.Join("testdata", "core-record-2.42.bin")) + if rerr != nil { + t.Fatalf("read golden: %v", rerr) + } + body := beI64(-1) // transactionID + body = append(body, byte(pageLargeTypeNone)) // largeMessageType + body = append(body, golden...) // [persisterID=1][core persister payload] + body = append(body, beI32(1)...) // queueIDsCount + body = append(body, beI64(7)...) // queueID 7 + entry := encodePageEntry(body) + root := writePageFile(t, "addr1", "000000001.page", entry) + + got, diag, err := ReadPaging(root, CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if diag.CoreSkipped != 0 || diag.UndecodableEntries != 0 { + t.Fatalf("diag = %+v, want zero", diag) + } + if len(got) != 1 { + t.Fatalf("want 1 exported paged core message, got %d", len(got)) + } + if got[0].Core == nil || got[0].Core.Address != "salvage.core" { + t.Errorf("Core payload not decoded: %+v", got[0].Core) + } + if len(got[0].QueueIDs) != 1 || got[0].QueueIDs[0] != 7 { + t.Errorf("QueueIDs = %v, want [7]", got[0].QueueIDs) + } +} + +func TestReadPagingCoreMalformedUndecodable(t *testing.T) { + body := beI64(-1) // transactionID + body = append(body, byte(pageLargeTypeNone)) // largeMessageType + body = append(body, persisterCoreMessage) // persister id + body = append(body, []byte{0xDE, 0xAD, 0xBE, 0xEF}...) // truncated core payload + entry := encodePageEntry(body) + root := writePageFile(t, "addr1", "000000001.page", entry) + + got, diag, err := ReadPaging(root, CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(got) != 0 { + t.Fatalf("want 0 messages, got %d: %+v", len(got), got) + } + if diag.UndecodableEntries != 1 { + t.Errorf("UndecodableEntries = %d, want 1", diag.UndecodableEntries) + } +} + +func TestReadPagingCoreLargeTypeSkipped(t *testing.T) { + // largeMessageType = CORE (1): a Core-protocol large-message header this + // AMQP-only reader cannot decode, per format_notes.md section 8's "if + // type in {CORE, OLD_CORE}" branch. Grouped with persister-id Core under + // the same CoreSkipped counter (both mean "cannot decode this entry"). + body := beI64(-1) + body = append(body, byte(pageLargeTypeCore)) + body = append(body, 0x01, 0x02, 0x03) // opaque core-large header, never parsed + entry := encodePageEntry(body) + root := writePageFile(t, "addr1", "000000001.page", entry) + + got, diag, err := ReadPaging(root, CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(got) != 0 { + t.Fatalf("want 0 messages, got %d: %+v", len(got), got) + } + if diag.CoreSkipped != 1 { + t.Errorf("CoreSkipped = %d, want 1", diag.CoreSkipped) + } +} + +// --- ReadPaging: defensive persister cases (correction #4: no paged-large +// entries in the fixture, but handle defensively) --- + +func TestReadPagingLargePersisterSkipped(t *testing.T) { + body := beI64(-1) + body = append(body, byte(pageLargeTypeNotCore)) + body = append(body, persisterAMQPLargeMessage) + entry := encodePageEntry(body) + root := writePageFile(t, "addr1", "000000001.page", entry) + + got, diag, err := ReadPaging(root, CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(got) != 0 { + t.Fatalf("want 0 messages, got %d: %+v", len(got), got) + } + if diag.LargeSkipped != 1 { + t.Errorf("LargeSkipped = %d, want 1", diag.LargeSkipped) + } +} + +func TestReadPagingUnknownPersisterCounted(t *testing.T) { + body := beI64(-1) + body = append(body, byte(pageLargeTypeNotCore)) + body = append(body, 99) // not a recognized persister id + entry := encodePageEntry(body) + root := writePageFile(t, "addr1", "000000001.page", entry) + + got, diag, err := ReadPaging(root, CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(got) != 0 { + t.Fatalf("want 0 messages, got %d: %+v", len(got), got) + } + if diag.UnknownPersister != 1 { + t.Errorf("UnknownPersister = %d, want 1", diag.UnknownPersister) + } +} + +// --- ReadPaging: misc --- + +func TestReadPagingNonexistentDir(t *testing.T) { + got, diag, err := ReadPaging(filepath.Join(t.TempDir(), "nonexistent"), CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(got) != 0 { + t.Errorf("got = %v, want empty", got) + } + if diag.CoreSkipped != 0 || diag.LargeSkipped != 0 || diag.UnknownPersister != 0 || + diag.UndecodableEntries != 0 || len(diag.CorruptPages) != 0 || diag.PagesSkippedComplete != 0 { + t.Errorf("diag = %+v, want zero", diag) + } +} + +func TestReadPagingEmptyAddressDir(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "addr1"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + got, _, err := ReadPaging(root, CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(got) != 0 { + t.Errorf("got = %v, want empty", got) + } +} + +// --- ReadPaging: fixture (brief step 1a) --- + +// fixtureCursorState replays the fixture's message journal through +// BuildCursorState. The fixture has zero ACKNOWLEDGE_CURSOR/ +// PAGE_CURSOR_COMPLETE survivors (format_notes.md section 3's census: only +// a PAGE_CURSOR_COUNTER_VALUE bookkeeping record is present), so this is +// expected to yield an effectively-empty CursorState -- exercised here so +// the fixture test runs the real pipe end to end rather than hand-building +// an empty CursorState{}. +func fixtureCursorState(t *testing.T, dataDir string) CursorState { + t.Helper() + p := NewReplayer() + jdiags, err := ReadJournalDir(filepath.Join(dataDir, "journal"), "activemq-data", "amq", p.Feed) + if err != nil { + t.Fatalf("ReadJournalDir: %v", err) + } + if len(jdiags) != 0 { + t.Fatalf("journal diags: %+v", jdiags) + } + survivors, _ := p.Resolve() + return BuildCursorState(survivors) +} + +// TestReadPagingFixture covers format_notes.md section 8's fixture census: +// 456 of the 500 salvage.paged manifest entries live in page files (the +// other 44 are journal-resident, covered by +// TestDecodeMessagesFixturePagedSpillover in message_test.go; 456+44=500). +// Nothing was consumed against salvage.paged in the fixture, so every page +// entry round-trips and no cursor filtering should trigger. +func TestReadPagingFixture(t *testing.T) { + dir := fixtureDir(t) + cursors := fixtureCursorState(t, dir) + + names, bdiags, err := ReadQueueBindings(filepath.Join(dir, "bindings")) + if err != nil { + t.Fatalf("ReadQueueBindings: %v", err) + } + if len(bdiags) != 0 { + t.Fatalf("bindings diags: %+v", bdiags) + } + var pagedQueueID int64 = -1 + for id, name := range names { + if name == "salvage.paged" { + pagedQueueID = id + } + } + if pagedQueueID == -1 { + t.Fatalf("no salvage.paged binding in %v", names) + } + + messages, diag, err := ReadPaging(filepath.Join(dir, "paging"), cursors) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if len(messages) != 456 { + t.Fatalf("want 456 paged messages, got %d", len(messages)) + } + if diag.CoreSkipped != 0 || diag.LargeSkipped != 0 || diag.UnknownPersister != 0 || + diag.UndecodableEntries != 0 || len(diag.CorruptPages) != 0 || diag.PagesSkippedComplete != 0 { + t.Errorf("diag = %+v, want zero", diag) + } + + man := loadManifest(t) + want := man.Queues["salvage.paged"] + wantLenByHash := make(map[string]int, len(want)) + for _, e := range want { + wantLenByHash[e.BodySha256] = e.BodyLen + } + seen := make(map[string]bool, len(messages)) + + for _, m := range messages { + if len(m.QueueIDs) != 1 || m.QueueIDs[0] != pagedQueueID { + t.Fatalf("QueueIDs = %v, want [%d]", m.QueueIDs, pagedQueueID) + } + // PagedMessage.AMQP is the raw AMQP-encoded message; extract the + // Data section the same way message_test.go's verifyQueueBodies + // does, via go-amqp, so this cross-checks against the manifest's + // body-only sha256/len. + body, err := amqpDataSection(m.AMQP) + if err != nil { + t.Fatalf("unmarshal AMQP: %v", err) + } + hash := sha256Hex(body) + wantLen, ok := wantLenByHash[hash] + if !ok { + t.Fatalf("body sha256 %s not present in manifest", hash) + } + if wantLen != len(body) { + t.Errorf("body len = %d, want %d", len(body), wantLen) + } + if seen[hash] { + t.Errorf("body sha256 %s seen more than once", hash) + } + seen[hash] = true + } +} + +// --- OOM guard regression test --- + +func TestReadPagingHugeQueueCountRegression(t *testing.T) { + // A corrupt queueIDsCount (e.g., 0x7FFFFFFF) should not allocate ~17GB. + // Build an entry with a huge queueIDsCount that would overflow if not guarded. + payload := beI64(-1) // transactionID + payload = append(payload, byte(pageLargeTypeNone)) // largeMessageType + payload = append(payload, persisterAMQPMessageV3) + payload = append(payload, beI64(77)...) // messageID + payload = append(payload, beI64(0)...) // messageFormat + payload = append(payload, encodeNullableSimpleString("test.addr", true)...) + payload = append(payload, beI32(5)...) // amqp size + payload = append(payload, []byte("hello")...) // 5 bytes of AMQP + payload = append(payload, beI32(0)...) // extraPropsSize + payload = append(payload, beI64(0)...) // expiration (V3) + // Corrupt queueIDsCount: 0x7FFFFFFF (huge value that would preallocate ~17GB) + payload = append(payload, beI32(0x7FFFFFFF)...) + + // Place it in a page with a good entry before it. + goodPayload := buildPagedMessagePayload(persisterAMQPMessageV3, []byte("good-body"), []int64{1}) + page := append(encodePageEntry(goodPayload), encodePageEntry(payload)...) + root := writePageFile(t, "addr1", "000000001.page", page) + + got, diag, err := ReadPaging(root, CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + + // The good entry should be kept; the corrupt one should be skipped. + if len(got) != 1 { + t.Fatalf("want 1 surviving entry (first one good, second corrupt), got %d: %+v", len(got), got) + } + if string(got[0].AMQP) != "good-body" { + t.Errorf("AMQP = %q, want good-body", got[0].AMQP) + } + + // The corrupt entry should increment UndecodableEntries, not cause a panic or massive allocation. + if diag.UndecodableEntries != 1 { + t.Errorf("UndecodableEntries = %d, want 1", diag.UndecodableEntries) + } +} + +// --- PagingReferenced tests --- + +func TestPagingReferencedDetectsPagingUserTypes(t *testing.T) { + // Table-driven: each of the six paging userTypes should return true. + tests := []struct { + name string + userType byte + want bool + }{ + {"PAGE_TRANSACTION", PageTransaction, true}, + {"ACKNOWLEDGE_CURSOR", AcknowledgeCursor, true}, + {"PAGE_CURSOR_COUNTER_VALUE", PageCursorCounterValue, true}, + {"PAGE_CURSOR_COUNTER_INC", PageCursorCounterInc, true}, + {"PAGE_CURSOR_COMPLETE", PageCursorComplete, true}, + {"PAGE_PENDING_COUNTER", PageCursorPendingCounter, true}, + {"ADD_MESSAGE_PROTOCOL", AddMessageProtocol, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + survivors := []Survivor{ + {ID: 1, UserType: tt.userType, Body: []byte{1, 2, 3}}, + } + if got := PagingReferenced(survivors); got != tt.want { + t.Errorf("PagingReferenced = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPagingReferencedEmptyInput(t *testing.T) { + if got := PagingReferenced(nil); got { + t.Errorf("PagingReferenced(nil) = %v, want false", got) + } + if got := PagingReferenced([]Survivor{}); got { + t.Errorf("PagingReferenced(empty) = %v, want false", got) + } +} + +func TestReadPagingDirMissingFlag(t *testing.T) { + // Nonexistent dir should set DirMissing=true + _, diag, err := ReadPaging(filepath.Join(t.TempDir(), "nonexistent"), CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if !diag.DirMissing { + t.Errorf("DirMissing = %v, want true (dir does not exist)", diag.DirMissing) + } + + // Existing but empty dir should leave DirMissing=false + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "addr1"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + _, diag, err = ReadPaging(root, CursorState{}) + if err != nil { + t.Fatalf("ReadPaging: %v", err) + } + if diag.DirMissing { + t.Errorf("DirMissing = %v, want false (dir exists but is empty)", diag.DirMissing) + } +} diff --git a/internal/journal/replay.go b/internal/journal/replay.go new file mode 100644 index 0000000..35c4595 --- /dev/null +++ b/internal/journal/replay.go @@ -0,0 +1,215 @@ +package journal + +import ( + "fmt" + "sort" +) + +// Survivor is a record that outlived replay: its ADD plus every committed +// UPDATE applied to it, in order. +type Survivor struct { + ID int64 + UserType byte + Body []byte + Updates []RawRecord // committed UPDATE records, journal order +} + +// ReplayDiag counts what replay discarded. +type ReplayDiag struct { + RolledBack int // records in rolled-back txs + InDoubt int // records in prepared-but-uncommitted txs (spec: skipped, reported) + Deleted int +} + +// txBuffer holds the not-yet-resolved records of one open transaction, plus +// whether it has seen a PREPARE_RECORD. A tx that never sees a terminal +// record (COMMIT/ROLLBACK) by the time Resolve is called is treated as +// rolled back unless it was prepared, in which case it is in-doubt -- see +// Resolve. +type txBuffer struct { + records []RawRecord + prepared bool +} + +// Replayer folds a journal record stream into surviving records, honoring +// transactions: tx-scoped adds/updates/deletes count only under COMMIT; +// PREPARE without COMMIT is in-doubt; ROLLBACK (or a tx with no terminal +// record) discards. Feed the records in on-disk/journal order (as produced +// by ReadJournalDir), then call Resolve once the stream is exhausted. +// +// Pure logic -- no bytes, no I/O. +type Replayer struct { + // records holds the currently-live survivors, keyed by record ID. + records map[int64]*Survivor + + // orphanUpdates holds UPDATE (and EVENT) records that arrived for an ID + // with no live ADD yet -- normal at file boundaries (the ADD lived in an + // earlier, since-compacted file) or as a compaction artifact where the + // compactor wrote an update ahead of its add in the rewritten file. They + // are applied to the survivor's Updates, in arrival order, the moment a + // matching ADD is seen; if no matching ADD ever arrives they are + // silently dropped in Resolve. + orphanUpdates map[int64][]RawRecord + + // txs holds buffered records for transactions not yet resolved by a + // terminal record (PREPARE does not resolve a tx; only COMMIT/ROLLBACK + // do). + txs map[int64]*txBuffer + + diag ReplayDiag +} + +// NewReplayer returns an empty Replayer ready for Feed. +func NewReplayer() *Replayer { + return &Replayer{ + records: make(map[int64]*Survivor), + orphanUpdates: make(map[int64][]RawRecord), + txs: make(map[int64]*txBuffer), + } +} + +// Feed is the emit callback for ReadJournalDir: it folds one raw journal +// record into replay state. Non-transactional ADD/UPDATE/DELETE apply +// immediately; transactional records are buffered under their TxID until a +// terminal record (COMMIT applies, ROLLBACK discards) resolves the +// transaction; PREPARE marks the transaction as prepared without resolving +// it (see Resolve for how unresolved transactions are counted). +func (p *Replayer) Feed(r RawRecord) error { + switch r.Type { + case AddRecord: + p.applyAdd(r) + case UpdateRecord, EventRecord: + // EVENT_RECORD shares ADD/UPDATE_RECORD's wire shape (record ID + + // user-typed body, format_notes.md section 3) and never appears in + // the harvested fixture; it is folded in alongside plain updates so + // an unanticipated use doesn't silently vanish from replay output. + p.applyUpdate(r) + case DeleteRecord: + p.applyDelete(r) + case AddRecordTx, UpdateRecordTx, DeleteRecordTx: + tx := p.txFor(r.TxID) + tx.records = append(tx.records, r) + case PrepareRecord: + p.txFor(r.TxID).prepared = true + case CommitRecord: + p.commitTx(r.TxID) + case RollbackRecord: + p.rollbackTx(r.TxID) + default: + // Unreachable in practice: every byte ReadJournalDir hands to emit + // has already been validated into [EventRecord, RollbackRecord] by + // readJournalFile/parseRecord, and every value in that range is + // handled above. + return fmt.Errorf("journal: replay: unexpected record type %d", r.Type) + } + return nil +} + +// Resolve finalizes replay: any transaction still open (no COMMIT/ROLLBACK +// ever arrived for it) is resolved now. Per format_notes.md section 9 +// (XmlDataExporter.processMessageJournal / Journal.load semantics): an +// unterminated transaction is discarded exactly like an explicit rollback, +// so it counts toward RolledBack; a transaction that reached PREPARE but +// never got a terminal record is in-doubt (XmlDataExporter discards +// preparedTransactions -- in-doubt messages are not exported, only counted). +// +// The returned survivors are ordered by ascending ID. +func (p *Replayer) Resolve() ([]Survivor, ReplayDiag) { + for _, tx := range p.txs { + if tx.prepared { + p.diag.InDoubt += len(tx.records) + } else { + p.diag.RolledBack += len(tx.records) + } + } + p.txs = make(map[int64]*txBuffer) + + out := make([]Survivor, 0, len(p.records)) + for _, sv := range p.records { + out = append(out, *sv) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, p.diag +} + +// txFor returns the buffer for txID, creating it if this is the first +// record seen for that transaction. +func (p *Replayer) txFor(txID int64) *txBuffer { + tx, ok := p.txs[txID] + if !ok { + tx = &txBuffer{} + p.txs[txID] = tx + } + return tx +} + +// commitTx applies every buffered record of txID, in the order it was fed, +// then drops the transaction. A COMMIT for a TxID with no buffered records +// (e.g. an empty or already-resolved transaction) is a no-op. +func (p *Replayer) commitTx(txID int64) { + tx, ok := p.txs[txID] + if !ok { + return + } + for _, r := range tx.records { + switch r.Type { + case AddRecordTx: + p.applyAdd(r) + case UpdateRecordTx: + p.applyUpdate(r) + case DeleteRecordTx: + p.applyDelete(r) + } + } + delete(p.txs, txID) +} + +// rollbackTx discards every buffered record of txID, counting them into +// RolledBack, then drops the transaction. A ROLLBACK for a TxID with no +// buffered records is a no-op. +func (p *Replayer) rollbackTx(txID int64) { + tx, ok := p.txs[txID] + if !ok { + return + } + p.diag.RolledBack += len(tx.records) + delete(p.txs, txID) +} + +// applyAdd installs a new survivor for r.ID, folding in any updates that +// arrived for this ID before its add (orphaned at the time, now resolved). +// A later ADD for an ID that already has a live survivor replaces it +// outright (this reader has not observed that case in practice, but it +// mirrors non-transactional semantics: the newest add for an ID wins). +func (p *Replayer) applyAdd(r RawRecord) { + sv := &Survivor{ID: r.ID, UserType: r.UserType, Body: r.Body} + if pending, ok := p.orphanUpdates[r.ID]; ok { + sv.Updates = append(sv.Updates, pending...) + delete(p.orphanUpdates, r.ID) + } + p.records[r.ID] = sv +} + +// applyUpdate appends r to its target survivor's Updates if the survivor is +// already live, or buffers it as an orphan (see orphanUpdates) if the add +// has not been seen yet. +func (p *Replayer) applyUpdate(r RawRecord) { + if sv, ok := p.records[r.ID]; ok { + sv.Updates = append(sv.Updates, r) + return + } + p.orphanUpdates[r.ID] = append(p.orphanUpdates[r.ID], r) +} + +// applyDelete removes r.ID's survivor, if any, counting it into Deleted. A +// delete for an ID with no live survivor (e.g. its add lived in an earlier, +// since-compacted file) is a no-op: there is nothing to discard, and any +// orphan updates queued for that ID are dropped too since no add can ever +// resurrect them. +func (p *Replayer) applyDelete(r RawRecord) { + if _, ok := p.records[r.ID]; ok { + delete(p.records, r.ID) + p.diag.Deleted++ + } + delete(p.orphanUpdates, r.ID) +} diff --git a/internal/journal/replay_test.go b/internal/journal/replay_test.go new file mode 100644 index 0000000..f2516b6 --- /dev/null +++ b/internal/journal/replay_test.go @@ -0,0 +1,371 @@ +package journal + +import ( + "path/filepath" + "testing" +) + +// findSurvivor returns the survivor with the given id, or nil. +func findSurvivor(survivors []Survivor, id int64) *Survivor { + for i := range survivors { + if survivors[i].ID == id { + return &survivors[i] + } + } + return nil +} + +func TestReplayAddSurvives(t *testing.T) { + p := NewReplayer() + if err := p.Feed(RawRecord{Type: AddRecord, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}); err != nil { + t.Fatalf("Feed: %v", err) + } + survivors, diag := p.Resolve() + if len(survivors) != 1 { + t.Fatalf("want 1 survivor, got %d", len(survivors)) + } + sv := survivors[0] + if sv.ID != 1 || string(sv.Body) != "body" || sv.UserType != AddMessageProtocol { + t.Errorf("survivor = %+v, want ID=1 Body=body UserType=AddMessageProtocol", sv) + } + if len(sv.Updates) != 0 { + t.Errorf("want 0 updates, got %d", len(sv.Updates)) + } + if diag != (ReplayDiag{}) { + t.Errorf("diag = %+v, want zero", diag) + } +} + +func TestReplayAddThenDeleteGone(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecord, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + mustFeed(t, p, RawRecord{Type: DeleteRecord, ID: 1}) + + survivors, diag := p.Resolve() + if len(survivors) != 0 { + t.Fatalf("want 0 survivors, got %d: %+v", len(survivors), survivors) + } + if diag.Deleted != 1 { + t.Errorf("diag.Deleted = %d, want 1", diag.Deleted) + } +} + +func TestReplayAddThenUpdateSurvivesWithUpdate(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecord, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + upd := RawRecord{Type: UpdateRecord, ID: 1, UserType: AddRef, Body: []byte("ref")} + mustFeed(t, p, upd) + + survivors, _ := p.Resolve() + sv := findSurvivor(survivors, 1) + if sv == nil { + t.Fatalf("survivor 1 not found") + } + if len(sv.Updates) != 1 || string(sv.Updates[0].Body) != "ref" { + t.Errorf("sv.Updates = %+v, want 1 update with body 'ref'", sv.Updates) + } +} + +func TestReplayTxAddCommitSurvives(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecordTx, TxID: 100, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + mustFeed(t, p, RawRecord{Type: CommitRecord, TxID: 100}) + + survivors, diag := p.Resolve() + if len(survivors) != 1 { + t.Fatalf("want 1 survivor, got %d", len(survivors)) + } + if diag.RolledBack != 0 || diag.InDoubt != 0 { + t.Errorf("diag = %+v, want zero RolledBack/InDoubt", diag) + } +} + +func TestReplayTxAddRollbackGone(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecordTx, TxID: 100, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + mustFeed(t, p, RawRecord{Type: RollbackRecord, TxID: 100}) + + survivors, diag := p.Resolve() + if len(survivors) != 0 { + t.Fatalf("want 0 survivors, got %d: %+v", len(survivors), survivors) + } + if diag.RolledBack != 1 { + t.Errorf("diag.RolledBack = %d, want 1", diag.RolledBack) + } +} + +// TestReplayTxNoTerminalRecordDiscarded covers a transaction with buffered +// records but no COMMIT/ROLLBACK/PREPARE at all by end of stream (e.g. the +// broker crashed mid-transaction, so the journal simply has no terminal +// record for it). format_notes.md section 9 (citing +// XmlDataExporter.processMessageJournal / Journal.load): "ROLLBACK and +// **unterminated** transactions are discarded" -- an unterminated tx is +// treated exactly like an explicit rollback, so this counts toward +// RolledBack, not InDoubt (InDoubt is reserved for transactions that reached +// PREPARE but never got a COMMIT/ROLLBACK). +func TestReplayTxNoTerminalRecordDiscarded(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecordTx, TxID: 100, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + + survivors, diag := p.Resolve() + if len(survivors) != 0 { + t.Fatalf("want 0 survivors, got %d: %+v", len(survivors), survivors) + } + if diag.RolledBack != 1 { + t.Errorf("diag.RolledBack = %d, want 1", diag.RolledBack) + } + if diag.InDoubt != 0 { + t.Errorf("diag.InDoubt = %d, want 0", diag.InDoubt) + } +} + +// TestReplayTxPrepareOnlyInDoubt: format_notes.md section 9 -- "PREPARE-only +// tx go into preparedTransactions (in-doubt)... XmlDataExporter discards +// preparedTransactions (in-doubt messages are NOT exported)." +func TestReplayTxPrepareOnlyInDoubt(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecordTx, TxID: 100, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + mustFeed(t, p, RawRecord{Type: PrepareRecord, TxID: 100, NumberOfRecords: 1}) + + survivors, diag := p.Resolve() + if len(survivors) != 0 { + t.Fatalf("want 0 survivors, got %d: %+v", len(survivors), survivors) + } + if diag.InDoubt != 1 { + t.Errorf("diag.InDoubt = %d, want 1", diag.InDoubt) + } + if diag.RolledBack != 0 { + t.Errorf("diag.RolledBack = %d, want 0", diag.RolledBack) + } +} + +func TestReplayTxPrepareThenCommitSurvives(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecordTx, TxID: 100, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + mustFeed(t, p, RawRecord{Type: PrepareRecord, TxID: 100, NumberOfRecords: 1}) + mustFeed(t, p, RawRecord{Type: CommitRecord, TxID: 100}) + + survivors, diag := p.Resolve() + if len(survivors) != 1 { + t.Fatalf("want 1 survivor, got %d", len(survivors)) + } + if diag.InDoubt != 0 || diag.RolledBack != 0 { + t.Errorf("diag = %+v, want zero", diag) + } +} + +// TestReplayTxPrepareThenRollbackDiscarded covers a heuristic rollback of a +// prepared (2PC) transaction: it must count as RolledBack, not InDoubt -- +// InDoubt is only for prepared transactions that never got a terminal +// COMMIT/ROLLBACK at all. +func TestReplayTxPrepareThenRollbackDiscarded(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecordTx, TxID: 100, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + mustFeed(t, p, RawRecord{Type: PrepareRecord, TxID: 100, NumberOfRecords: 1}) + mustFeed(t, p, RawRecord{Type: RollbackRecord, TxID: 100}) + + survivors, diag := p.Resolve() + if len(survivors) != 0 { + t.Fatalf("want 0 survivors, got %d: %+v", len(survivors), survivors) + } + if diag.RolledBack != 1 { + t.Errorf("diag.RolledBack = %d, want 1", diag.RolledBack) + } + if diag.InDoubt != 0 { + t.Errorf("diag.InDoubt = %d, want 0", diag.InDoubt) + } +} + +func TestReplayDeleteInTxCommittedGone(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecord, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + mustFeed(t, p, RawRecord{Type: DeleteRecordTx, TxID: 100, ID: 1}) + mustFeed(t, p, RawRecord{Type: CommitRecord, TxID: 100}) + + survivors, diag := p.Resolve() + if len(survivors) != 0 { + t.Fatalf("want 0 survivors, got %d: %+v", len(survivors), survivors) + } + if diag.Deleted != 1 { + t.Errorf("diag.Deleted = %d, want 1", diag.Deleted) + } +} + +func TestReplayDeleteInTxRolledBackSurvives(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecord, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + mustFeed(t, p, RawRecord{Type: DeleteRecordTx, TxID: 100, ID: 1}) + mustFeed(t, p, RawRecord{Type: RollbackRecord, TxID: 100}) + + survivors, diag := p.Resolve() + if len(survivors) != 1 { + t.Fatalf("want 1 survivor, got %d: %+v", len(survivors), survivors) + } + if survivors[0].ID != 1 { + t.Errorf("survivor ID = %d, want 1", survivors[0].ID) + } + if diag.RolledBack != 1 { + t.Errorf("diag.RolledBack = %d, want 1", diag.RolledBack) + } + if diag.Deleted != 0 { + t.Errorf("diag.Deleted = %d, want 0", diag.Deleted) + } +} + +// TestReplayUpdateBeforeAddAppliedWhenAddSurvives covers a compaction +// artifact (format_notes.md section 9 bullet 3 / section on paging +// cross-check): compaction can rewrite a message's UPDATE (e.g. its ADD_REF) +// ahead of its ADD_RECORD in the resulting file. The update must still land +// on the survivor once the add arrives. +func TestReplayUpdateBeforeAddAppliedWhenAddSurvives(t *testing.T) { + p := NewReplayer() + upd := RawRecord{Type: UpdateRecord, ID: 1, UserType: AddRef, Body: []byte("ref")} + mustFeed(t, p, upd) + mustFeed(t, p, RawRecord{Type: AddRecord, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + + survivors, _ := p.Resolve() + sv := findSurvivor(survivors, 1) + if sv == nil { + t.Fatalf("survivor 1 not found") + } + if len(sv.Updates) != 1 || string(sv.Updates[0].Body) != "ref" { + t.Errorf("sv.Updates = %+v, want 1 update with body 'ref'", sv.Updates) + } +} + +// TestReplayUpdateBeforeAddDroppedWhenAddNeverArrives covers the other half +// of the same compaction scenario: the update's target ADD lived in an +// earlier, since-compacted file and never shows up in this stream at all +// (normal at file boundaries). The orphan update must be silently dropped, +// not surfaced as a survivor or a diagnostic. +func TestReplayUpdateBeforeAddDroppedWhenAddNeverArrives(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: UpdateRecord, ID: 1, UserType: AddRef, Body: []byte("ref")}) + + survivors, diag := p.Resolve() + if len(survivors) != 0 { + t.Fatalf("want 0 survivors, got %d: %+v", len(survivors), survivors) + } + if diag != (ReplayDiag{}) { + t.Errorf("diag = %+v, want zero (orphan update is dropped silently)", diag) + } +} + +func TestReplayUpdatesPreserveJournalOrder(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecord, ID: 1, UserType: AddMessageProtocol, Body: []byte("body")}) + mustFeed(t, p, RawRecord{Type: UpdateRecord, ID: 1, UserType: AddRef, Body: []byte("first")}) + mustFeed(t, p, RawRecord{Type: UpdateRecord, ID: 1, UserType: SetScheduledDeliveryTime, Body: []byte("second")}) + + survivors, _ := p.Resolve() + sv := findSurvivor(survivors, 1) + if sv == nil { + t.Fatalf("survivor 1 not found") + } + if len(sv.Updates) != 2 || string(sv.Updates[0].Body) != "first" || string(sv.Updates[1].Body) != "second" { + t.Errorf("sv.Updates = %+v, want [first, second] in order", sv.Updates) + } +} + +func TestReplayResultOrderedByAscendingID(t *testing.T) { + p := NewReplayer() + mustFeed(t, p, RawRecord{Type: AddRecord, ID: 5, UserType: AddMessageProtocol, Body: []byte("five")}) + mustFeed(t, p, RawRecord{Type: AddRecord, ID: 1, UserType: AddMessageProtocol, Body: []byte("one")}) + mustFeed(t, p, RawRecord{Type: AddRecord, ID: 3, UserType: AddMessageProtocol, Body: []byte("three")}) + + survivors, _ := p.Resolve() + if len(survivors) != 3 { + t.Fatalf("want 3 survivors, got %d", len(survivors)) + } + var ids []int64 + for _, sv := range survivors { + ids = append(ids, sv.ID) + } + want := []int64{1, 3, 5} + for i, id := range ids { + if id != want[i] { + t.Errorf("survivors[%d].ID = %d, want %d (order = %v)", i, id, want[i], ids) + } + } +} + +func mustFeed(t *testing.T, p *Replayer, r RawRecord) { + t.Helper() + if err := p.Feed(r); err != nil { + t.Fatalf("Feed(%+v): %v", r, err) + } +} + +// TestReplayFixtureMessageJournal replays the harvested 2.42 fixture's +// message journal end to end (ReadJournalDir -> Feed -> Resolve). The broker +// shut down clean, so there are no ROLLBACK or PREPARE records here -- the +// transaction paths above are synthetic by necessity; this covers the happy +// path with real bytes. +// +// Expected numbers (manifest.json + format_notes.md fixture census): +// - 59 ADD_MESSAGE_PROTOCOL (45) adds reach the journal: 5 plain + 5 props +// - 1 scheduled + 1 large + 3 acked + 44 of the 500 paged messages +// (paging is a spillover, not a mirror; the other 456 live in page +// files, out of scope for journal replay). +// - The 3 salvage.acked messages were received+accepted, leaving 3 +// DELETE_RECORDs => Deleted = 3, and 59-3 = 56 message survivors. +// - Every surviving message carries its ADD_REF (userType 32), which +// arrives framed as an UPDATE_RECORD. +// - Exactly 1 survivor (salvage.scheduled) carries a +// SET_SCHEDULED_DELIVERY_TIME (36) update. +func TestReplayFixtureMessageJournal(t *testing.T) { + dir := fixtureDir(t) + + p := NewReplayer() + diags, err := ReadJournalDir(filepath.Join(dir, "journal"), "activemq-data", "amq", p.Feed) + if err != nil { + t.Fatalf("ReadJournalDir: %v", err) + } + if len(diags) != 0 { + t.Fatalf("want 0 diags over the clean fixture, got %d: %+v", len(diags), diags) + } + + survivors, diag := p.Resolve() + + if diag.Deleted != 3 { + t.Errorf("diag.Deleted = %d, want 3 (the fully-acked salvage.acked messages)", diag.Deleted) + } + if diag.RolledBack != 0 { + t.Errorf("diag.RolledBack = %d, want 0 (clean shutdown, no rolled-back txs)", diag.RolledBack) + } + if diag.InDoubt != 0 { + t.Errorf("diag.InDoubt = %d, want 0 (clean shutdown, no prepared txs)", diag.InDoubt) + } + + var messages, withRef, withScheduled int + for _, sv := range survivors { + if sv.UserType != AddMessageProtocol { + continue + } + messages++ + var hasRef, hasScheduled bool + for _, u := range sv.Updates { + switch u.UserType { + case AddRef: + hasRef = true + case SetScheduledDeliveryTime: + hasScheduled = true + } + } + if hasRef { + withRef++ + } + if hasScheduled { + withScheduled++ + } + } + if messages != 56 { + t.Errorf("AddMessageProtocol survivors = %d, want 56 (59 journal adds - 3 acked)", messages) + } + if withRef != messages { + t.Errorf("survivors with an AddRef update = %d, want all %d", withRef, messages) + } + if withScheduled != 1 { + t.Errorf("survivors with a SetScheduledDeliveryTime update = %d, want 1 (salvage.scheduled)", withScheduled) + } +} diff --git a/internal/journal/salvage.go b/internal/journal/salvage.go new file mode 100644 index 0000000..aca5f83 --- /dev/null +++ b/internal/journal/salvage.go @@ -0,0 +1,428 @@ +package journal + +import ( + "errors" + "fmt" + "io/fs" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/Azure/go-amqp" + "github.com/martikan/artemisctl/internal/store" +) + +// Options locates the four data sub-dirs (already resolved by the CLI). +type Options struct{ Bindings, Journal, LargeMessages, Paging string } + +// Summary is everything the CLI prints and gates exit codes on. +type Summary struct { + PerQueue map[string]int // exported records per queue name + Large, Paged int // Large = records from large-message-sourced messages; Paged = records read from page files (not journal-spilled paged messages) + LargestBytes int64 + Core int // exported Core-protocol records (subset of the per-queue totals) + Skips []string // human-readable, one per skip class w/ counts+ids + Diags []string // corruption/diagnostic notes (non-skip) + + // corrupt counts how many of the entries in Diags represent + // corruption-class incidents (structural journal/page/bindings damage -- + // check-size mismatch, truncated record, broken page-entry framing, an + // undecodable bindings-record body) as opposed to benign notes (missing + // large-messages/paging dirs, unknown-queue fallback, orphaned + // large-message files, or a fileID-mismatch "reuse leftover" from a + // normally-reused journal file). Populated exclusively by + // appendFileDiags, which reads the classification straight off each + // source FileDiag's Corrupt field -- never by matching against the + // assembled Diags prose. Unexported: only this package ever needs to set + // it, and Salvage's caller reads it via HasCorruption. + corrupt int +} + +// Total returns the total number of exported records across all queues. +func (s Summary) Total() int { + total := 0 + for _, n := range s.PerQueue { + total += n + } + return total +} + +// HasSkips reports whether anything was skipped (lost) during the run — the +// CLI gates its non-zero exit code on this (spec §1: "a recovery tool must +// not silently lose messages"). +func (s Summary) HasSkips() bool { + return len(s.Skips) > 0 +} + +// HasCorruption reports whether any Diags entry represents corruption-class +// structural damage (see the corrupt field's doc comment). The CLI gates its +// non-zero exit code on this exactly like HasSkips, and with the same +// --allow-skips override: corrupted journal/page/bindings data is data loss +// just as surely as an unsupported record type, and a recovery tool must not +// let that be missable in a script (spec §1/§5). +func (s Summary) HasCorruption() bool { + return s.corrupt > 0 +} + +// appendFileDiags formats diags (from ReadJournalDir/ReadPaging/ +// ReadQueueBindings) into s.Diags, prefixed with source (e.g. "bindings +// journal", "message journal", "paging") to disambiguate which reader hit +// the incident, and tallies how many are corruption-class into s.corrupt so +// HasCorruption sees them. Classification comes straight from each source +// FileDiag's own Corrupt field, set at the point the diag was created -- +// not from matching against the prose assembled here. +func (s *Summary) appendFileDiags(source string, diags []FileDiag) { + for _, d := range diags { + s.Diags = append(s.Diags, fmt.Sprintf("%s: %s (offset %d): %s", source, d.Path, d.Offset, d.Reason)) + if d.Corrupt { + s.corrupt++ + } + } +} + +// Salvage runs the full offline pipeline -- ReadQueueBindings -> +// ReadJournalDir(journal) -> Replayer -> DecodeMessages -> AttachLargeBodies +// -> ReadPaging -- and emits one store.Record per surviving (message, queue) +// pair, fanning a message referenced by N queues out into N records (spec +// §3). Records are emitted via emit as they are produced; an error from emit +// aborts the run immediately, returning the Summary accumulated so far +// alongside the error. +// +// Missing/unreadable Bindings or Journal dirs are fatal (spec §5). Missing +// LargeMessages or Paging dirs are not fatal -- AttachLargeBodies/ReadPaging +// already treat them as "nothing to attach/nothing was paged" -- but are +// worth a Diags warning when the journal actually references large messages +// or paging state, since an operator seeing a suspiciously low large/paged +// count needs to know the dir was simply absent rather than everything +// having failed to decode. +func Salvage(opts Options, emit func(store.Record) error) (Summary, error) { + summary := Summary{PerQueue: make(map[string]int)} + + if err := requireDir(opts.Bindings); err != nil { + return summary, fmt.Errorf("salvage: bindings dir: %w", err) + } + if err := requireDir(opts.Journal); err != nil { + return summary, fmt.Errorf("salvage: journal dir: %w", err) + } + + names, bindingDiags, err := ReadQueueBindings(opts.Bindings) + if err != nil { + return summary, fmt.Errorf("salvage: read bindings: %w", err) + } + summary.appendFileDiags("bindings journal", bindingDiags) + + replayer := NewReplayer() + journalDiags, err := ReadJournalDir(opts.Journal, "activemq-data", "amq", replayer.Feed) + if err != nil { + return summary, fmt.Errorf("salvage: read journal: %w", err) + } + summary.appendFileDiags("message journal", journalDiags) + + survivors, replayDiag := replayer.Resolve() + if replayDiag.InDoubt > 0 { + summary.Skips = append(summary.Skips, fmt.Sprintf( + "in-doubt transaction records (prepared but never committed or rolled back): %d", replayDiag.InDoubt)) + } + + messages, msgDiag, err := DecodeMessages(survivors) + if err != nil { + return summary, fmt.Errorf("salvage: decode messages: %w", err) + } + summary.Skips = append(summary.Skips, messageDiagSkips(msgDiag)...) + + // spec §5: warn (Diags only, not a Skip) if the journal references large + // messages but the large-messages dir is missing. Checked against the + // pre-attach messages, since a missing dir makes AttachLargeBodies drop + // every Large message into MissingFile -- checking post-attach could + // wrongly conclude "not referenced" once none are left. + if anyLarge(messages) && dirMissing(opts.LargeMessages) { + summary.Diags = append(summary.Diags, fmt.Sprintf( + "large-messages dir %s is missing but the journal references large messages", opts.LargeMessages)) + } + + messages, largeDiag, err := AttachLargeBodies(messages, opts.LargeMessages) + if err != nil { + return summary, fmt.Errorf("salvage: attach large bodies: %w", err) + } + summary.LargestBytes = largeDiag.LargestBytes + if len(largeDiag.MissingFile) > 0 { + summary.Skips = append(summary.Skips, fmt.Sprintf( + "large-message body files missing: %d (ids: %s)", len(largeDiag.MissingFile), joinInt64s(largeDiag.MissingFile))) + } + if len(largeDiag.Orphans) > 0 { + sortedOrphans := append([]string(nil), largeDiag.Orphans...) + sort.Strings(sortedOrphans) + summary.Diags = append(summary.Diags, fmt.Sprintf( + "orphaned large-message files with no surviving journal record (normal): %d (%s)", + len(sortedOrphans), strings.Join(sortedOrphans, ", "))) + } + + cursors := BuildCursorState(survivors) + pagedMessages, pagingDiag, err := ReadPaging(opts.Paging, cursors) + if err != nil { + return summary, fmt.Errorf("salvage: read paging: %w", err) + } + if PagingReferenced(survivors) && pagingDiag.DirMissing { + summary.Diags = append(summary.Diags, fmt.Sprintf( + "paging dir %s is missing but the journal references paging state", opts.Paging)) + } + summary.appendFileDiags("paging", pagingDiag.CorruptPages) + if pagingDiag.PagesSkippedComplete > 0 { + summary.Diags = append(summary.Diags, fmt.Sprintf( + "pages skipped as fully consumed per cursor state (not a loss): %d", pagingDiag.PagesSkippedComplete)) + } + summary.Skips = append(summary.Skips, pagingDiagSkips(pagingDiag)...) + + // unknownQueues counts, per queueID, how many records were exported + // under the synthetic "unknown-queue-" fallback name (spec §5: the + // message is still saved, so this is a Diags warning, not a Skip). + unknownQueues := make(map[int64]int) + + for _, m := range messages { + if m.Core != nil { + if err := emitCoreFanout(m.Core, m.ScheduledMs, m.QueueIDs, false, names, unknownQueues, &summary, emit); err != nil { + return summary, fmt.Errorf("salvage: core message %d: %w", m.ID, err) + } + continue + } + if err := emitFanout(m.AMQP, m.ScheduledMs, m.QueueIDs, m.Large, false, names, unknownQueues, &summary, emit); err != nil { + return summary, fmt.Errorf("salvage: message %d: %w", m.ID, err) + } + } + for _, pm := range pagedMessages { + if pm.Core != nil { + if err := emitCoreFanout(pm.Core, pm.ScheduledMs, pm.QueueIDs, true, names, unknownQueues, &summary, emit); err != nil { + return summary, fmt.Errorf("salvage: paged core message: %w", err) + } + continue + } + if err := emitFanout(pm.AMQP, pm.ScheduledMs, pm.QueueIDs, false, true, names, unknownQueues, &summary, emit); err != nil { + return summary, fmt.Errorf("salvage: paged message: %w", err) + } + } + + if len(unknownQueues) > 0 { + ids := make([]int64, 0, len(unknownQueues)) + for id := range unknownQueues { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + for _, id := range ids { + summary.Diags = append(summary.Diags, fmt.Sprintf( + "queueID %d has no surviving binding; %d record(s) exported under synthetic queue name unknown-queue-%d", + id, unknownQueues[id], id)) + } + } + + return summary, nil +} + +// emitFanout unmarshals amqpBytes once (applying the scheduled-delivery +// annotation if scheduledMs != 0), then emits one store.Record per queueID +// in queueIDs, updating summary's per-queue/Large/Paged counters as it goes. +// It returns the first error from emit, if any, without emitting further +// records for this message. +func emitFanout(amqpBytes []byte, scheduledMs int64, queueIDs []int64, large, paged bool, names map[int64]string, unknownQueues map[int64]int, summary *Summary, emit func(store.Record) error) error { + preparedAMQP, msg, err := prepareMessage(amqpBytes, scheduledMs) + if err != nil { + return err + } + + for _, qid := range queueIDs { + queue := queueName(names, qid, unknownQueues) + rec := store.Record{ + UUID: store.DedupID(msg, queue), + Queue: queue, + DrainedAt: time.Now().UnixNano(), + AMQP: preparedAMQP, + } + if err := emit(rec); err != nil { + return err + } + summary.PerQueue[queue]++ + if large { + summary.Large++ + } + if paged { + summary.Paged++ + } + } + return nil +} + +// emitCoreFanout serializes a decoded Core message once and emits one +// KindCore store.Record per surviving queueID, updating summary's +// per-queue/Core/Paged counters. A non-zero scheduledMs (from a +// SET_SCHEDULED_DELIVERY_TIME journal update) is threaded into the payload as +// the synthetic property "_ARTX_SCHEDULED_MS" so broker/coreconvert can set +// the AMQP x-opt-delivery-time annotation on redelivery. Dedup ids come from +// DedupIDCore (the Core message has no amqp.Message form). +func emitCoreFanout(core *CorePayload, scheduledMs int64, queueIDs []int64, paged bool, names map[int64]string, unknownQueues map[int64]int, summary *Summary, emit func(store.Record) error) error { + if scheduledMs != 0 { + if core.Properties == nil { + core.Properties = map[string]any{} + } + core.Properties["_ARTX_SCHEDULED_MS"] = scheduledMs + } + payload := core.Encode() + + for _, qid := range queueIDs { + queue := queueName(names, qid, unknownQueues) + rec := store.Record{ + UUID: store.DedupIDCore(payload, queue), + Queue: queue, + DrainedAt: time.Now().UnixNano(), + Kind: store.KindCore, + CorePayload: payload, + } + if err := emit(rec); err != nil { + return err + } + summary.PerQueue[queue]++ + summary.Core++ + if paged { + summary.Paged++ + } + } + return nil +} + +// prepareMessage unmarshals amqpBytes once. For a non-scheduled message +// (scheduledMs == 0) it returns amqpBytes verbatim, unmodified, alongside the +// decoded message (used only for hashing). For a scheduled message it sets +// the "x-opt-delivery-time" message annotation to scheduledMs and +// re-marshals, returning the new bytes -- this is the only case that +// re-marshals; every other message's stored AMQP bytes are the original, +// untouched journal/large/paged bytes. +func prepareMessage(amqpBytes []byte, scheduledMs int64) ([]byte, *amqp.Message, error) { + msg := new(amqp.Message) + if err := msg.UnmarshalBinary(amqpBytes); err != nil { + return nil, nil, fmt.Errorf("unmarshal AMQP message: %w", err) + } + if scheduledMs == 0 { + return amqpBytes, msg, nil + } + + if msg.Annotations == nil { + msg.Annotations = amqp.Annotations{} + } + msg.Annotations["x-opt-delivery-time"] = scheduledMs + remarshaled, err := msg.MarshalBinary() + if err != nil { + return nil, nil, fmt.Errorf("marshal scheduled AMQP message: %w", err) + } + return remarshaled, msg, nil +} + +// queueName resolves queueID via names, falling back to the synthetic +// "unknown-queue-" name (spec §5) when the bindings journal has no +// surviving binding for it -- e.g. the queue was deleted after the message +// was enqueued but before the broker died. unknownQueues counts fallback +// uses per queueID for the caller's summary Diags. +func queueName(names map[int64]string, queueID int64, unknownQueues map[int64]int) string { + if name, ok := names[queueID]; ok { + return name + } + unknownQueues[queueID]++ + return fmt.Sprintf("unknown-queue-%d", queueID) +} + +// anyLarge reports whether any message in msgs is a Large message (body +// joined from data/large-messages/.msg) -- used to decide whether a +// missing LargeMessages dir is worth a Diags warning. +func anyLarge(msgs []Message) bool { + for _, m := range msgs { + if m.Large { + return true + } + } + return false +} + +// requireDir returns an error if dir does not exist, is unreadable, or is +// not a directory. +func requireDir(dir string) error { + fi, err := os.Stat(dir) + if err != nil { + return err + } + if !fi.IsDir() { + return fmt.Errorf("%s: not a directory", dir) + } + return nil +} + +// dirMissing reports whether dir does not exist. Any other stat error (e.g. +// a permission problem) is deliberately NOT treated as "missing" here -- +// AttachLargeBodies/ReadPaging already surface those as hard errors from +// Salvage, so this helper only needs to detect the specific "absent +// directory" case the spec calls non-fatal. +func dirMissing(dir string) bool { + _, err := os.Stat(dir) + if err == nil { + return false + } + return errors.Is(err, fs.ErrNotExist) +} + +// messageDiagSkips formats MessageDiag's skip classes (Core-protocol +// messages this AMQP-only reader cannot decode, unrecognized persister ids, +// undecodable bodies) into Summary.Skips entries. +func messageDiagSkips(diag MessageDiag) []string { + var out []string + if len(diag.CoreSkipped) > 0 { + ids := make([]int64, 0, len(diag.CoreSkipped)) + refs := 0 + for id, qids := range diag.CoreSkipped { + ids = append(ids, id) + refs += len(qids) + } + out = append(out, fmt.Sprintf( + "Core-protocol messages skipped in message journal (unsupported by this AMQP-only reader): %d messages, %d surviving queue refs (ids: %s)", + len(ids), refs, joinInt64s(ids))) + } + if diag.UnknownPersister > 0 { + out = append(out, fmt.Sprintf("unrecognized persister id in message journal: %d", diag.UnknownPersister)) + } + if diag.UndecodableBody > 0 { + out = append(out, fmt.Sprintf("undecodable message bodies in message journal: %d", diag.UndecodableBody)) + } + return out +} + +// pagingDiagSkips formats PagingDiag's skip classes (Core-protocol/AMQP +// large entries embedded in page files, unrecognized persister ids, +// undecodable entries) into Summary.Skips entries. PagesSkippedComplete and +// CorruptPages are handled separately by the caller -- the former is not a +// loss (already-consumed pages are correctly excluded) and the latter is +// file-diag material, not a skip class. +func pagingDiagSkips(diag PagingDiag) []string { + var out []string + if diag.CoreSkipped > 0 { + out = append(out, fmt.Sprintf("Core-protocol entries skipped in page files (unsupported by this AMQP-only reader): %d", diag.CoreSkipped)) + } + if diag.LargeSkipped > 0 { + out = append(out, fmt.Sprintf("AMQP large-message entries embedded in page files skipped (unsupported): %d", diag.LargeSkipped)) + } + if diag.UnknownPersister > 0 { + out = append(out, fmt.Sprintf("unrecognized persister id in page files: %d", diag.UnknownPersister)) + } + if diag.UndecodableEntries > 0 { + out = append(out, fmt.Sprintf("undecodable entries in page files: %d", diag.UndecodableEntries)) + } + return out +} + +// joinInt64s formats ids in ascending order as a comma-separated list, for +// Skips/Diags entries. +func joinInt64s(ids []int64) string { + sorted := append([]int64(nil), ids...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + parts := make([]string, len(sorted)) + for i, id := range sorted { + parts[i] = strconv.FormatInt(id, 10) + } + return strings.Join(parts, ", ") +} diff --git a/internal/journal/salvage_core_test.go b/internal/journal/salvage_core_test.go new file mode 100644 index 0000000..d40434b --- /dev/null +++ b/internal/journal/salvage_core_test.go @@ -0,0 +1,140 @@ +package journal + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/martikan/artemisctl/internal/store" +) + +// coreFixtureDir extracts testdata/artemis-2.42-core-data.tar.gz — a data dir +// harvested from a real Artemis 2.42.0 broker that was fed Core-protocol +// messages (3 standard BYTES messages on salvage.core + 1 300 KiB large +// message on salvage.corelarge, via `artemis producer --protocol CORE`). +func coreFixtureDir(t *testing.T) string { + t.Helper() + tarball := filepath.Join("testdata", "artemis-2.42-core-data.tar.gz") + f, err := os.Open(tarball) + if err != nil { + if os.IsNotExist(err) { + t.Skipf("core fixture %s missing", tarball) + } + t.Fatalf("open core fixture: %v", err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("gunzip: %v", err) + } + defer gz.Close() + dst := t.TempDir() + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("read tar: %v", err) + } + name := filepath.Clean(hdr.Name) + if strings.HasPrefix(name, "..") || filepath.IsAbs(name) { + t.Fatalf("unsafe path %q", hdr.Name) + } + path := filepath.Join(dst, name) + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", name, err) + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir parent %s: %v", name, err) + } + out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + t.Fatalf("create %s: %v", name, err) + } + if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // trusted committed fixture + out.Close() + t.Fatalf("copy %s: %v", name, err) + } + out.Close() + } + } + return filepath.Join(dst, "data") +} + +// TestSalvageCoreFixtureEndToEnd runs the whole offline pipeline over a real +// Core-message data dir and asserts every Core message is decoded, exported as +// a KindCore store record, its large body joined, and no Core skips remain. +func TestSalvageCoreFixtureEndToEnd(t *testing.T) { + dir := coreFixtureDir(t) + opts := Options{ + Bindings: filepath.Join(dir, "bindings"), + Journal: filepath.Join(dir, "journal"), + LargeMessages: filepath.Join(dir, "large-messages"), + Paging: filepath.Join(dir, "paging"), + } + + var recs []store.Record + summary, err := Salvage(opts, func(r store.Record) error { + recs = append(recs, r) + return nil + }) + if err != nil { + t.Fatalf("Salvage: %v", err) + } + + // 3 standard salvage.core + 1 salvage.corelarge = 4 Core messages, each on + // exactly one queue. + if summary.Core != 4 { + t.Errorf("summary.Core = %d, want 4", summary.Core) + } + if summary.HasSkips() { + t.Errorf("unexpected skips: %v", summary.Skips) + } + + kindCore, large := 0, 0 + byQueue := map[string]int{} + for _, r := range recs { + if r.Kind != store.KindCore { + t.Errorf("record on %s has kind %d, want KindCore", r.Queue, r.Kind) + continue + } + kindCore++ + byQueue[r.Queue]++ + p, derr := DecodeCorePayload(r.CorePayload) + if derr != nil { + t.Fatalf("DecodeCorePayload for %s: %v", r.Queue, derr) + } + if p.Type != CoreTypeBytes { + t.Errorf("%s: Core.Type = %d, want BYTES", r.Queue, p.Type) + } + if p.Large { + large++ + if len(p.Body) != 307200 { + t.Errorf("large body len = %d, want 307200", len(p.Body)) + } + } else if len(p.Body) != 120 { + t.Errorf("%s: standard body len = %d, want 120", r.Queue, len(p.Body)) + } + } + if kindCore != 4 { + t.Errorf("KindCore records = %d, want 4", kindCore) + } + if large != 1 { + t.Errorf("large core records = %d, want 1", large) + } + if byQueue["salvage.core"] != 3 { + t.Errorf("salvage.core count = %d, want 3", byQueue["salvage.core"]) + } + if byQueue["salvage.corelarge"] != 1 { + t.Errorf("salvage.corelarge count = %d, want 1", byQueue["salvage.corelarge"]) + } +} diff --git a/internal/journal/salvage_test.go b/internal/journal/salvage_test.go new file mode 100644 index 0000000..09428e8 --- /dev/null +++ b/internal/journal/salvage_test.go @@ -0,0 +1,381 @@ +package journal + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Azure/go-amqp" + "github.com/martikan/artemisctl/internal/store" +) + +// fixtureSalvageOptions returns Options pointing at fixtureDir(t)'s four +// sub-dirs. +func fixtureSalvageOptions(t *testing.T, dir string) Options { + t.Helper() + return Options{ + Bindings: filepath.Join(dir, "bindings"), + Journal: filepath.Join(dir, "journal"), + LargeMessages: filepath.Join(dir, "large-messages"), + Paging: filepath.Join(dir, "paging"), + } +} + +// --- golden fixture test (brief Step 1, assertions a-f) --- + +func TestSalvageFixtureGolden(t *testing.T) { + dir := fixtureDir(t) + opts := fixtureSalvageOptions(t, dir) + + var records []store.Record + summary, err := Salvage(opts, func(r store.Record) error { + records = append(records, r) + return nil + }) + if err != nil { + t.Fatalf("Salvage: %v", err) + } + + man := loadManifest(t) + + // (a) per-queue counts exactly match the manifest. + for q, entries := range man.Queues { + if got, want := summary.PerQueue[q], len(entries); got != want { + t.Errorf("PerQueue[%s] = %d, want %d", q, got, want) + } + } + for q := range summary.PerQueue { + if _, ok := man.Queues[q]; !ok { + t.Errorf("PerQueue has unexpected queue %s = %d", q, summary.PerQueue[q]) + } + } + if summary.Total() != 5+5+1+1+500 { + t.Errorf("Total() = %d, want %d", summary.Total(), 5+5+1+1+500) + } + + // (b) every record's AMQP unmarshals and its body sha256 is in the + // manifest for that queue. + hashesByQueue := make(map[string]map[string]bool, len(man.Queues)) + for q, entries := range man.Queues { + set := make(map[string]bool, len(entries)) + for _, e := range entries { + set[e.BodySha256] = true + } + hashesByQueue[q] = set + } + + var scheduledRecord *store.Record + for i := range records { + r := &records[i] + var am amqp.Message + if err := am.UnmarshalBinary(r.AMQP); err != nil { + t.Fatalf("unmarshal record AMQP (queue %s): %v", r.Queue, err) + } + body := am.GetData() + sum := sha256.Sum256(body) + hash := hex.EncodeToString(sum[:]) + set, ok := hashesByQueue[r.Queue] + if !ok || !set[hash] { + t.Errorf("record on queue %s has body sha256 %s not present in manifest", r.Queue, hash) + } + if r.Queue == "salvage.scheduled" { + if scheduledRecord != nil { + t.Fatalf("more than one salvage.scheduled record") + } + scheduledRecord = r + } + } + + // (c) UUIDs are unique across the run. + seen := make(map[[16]byte]string, len(records)) + for _, r := range records { + if prevQueue, ok := seen[r.UUID]; ok { + t.Errorf("duplicate UUID %x: queues %s and %s", r.UUID, prevQueue, r.Queue) + } + seen[r.UUID] = r.Queue + } + + // (d) scheduled record carries the annotation. + if scheduledRecord == nil { + t.Fatal("no salvage.scheduled record emitted") + } + var schedMsg amqp.Message + if err := schedMsg.UnmarshalBinary(scheduledRecord.AMQP); err != nil { + t.Fatalf("unmarshal scheduled record: %v", err) + } + gotMs, ok := schedMsg.Annotations["x-opt-delivery-time"] + if !ok { + t.Fatal("scheduled record missing x-opt-delivery-time annotation") + } + wantMs := man.Queues["salvage.scheduled"][0].ScheduledAtMs + if fmt.Sprint(gotMs) != fmt.Sprint(wantMs) { + t.Errorf("x-opt-delivery-time = %v, want %v", gotMs, wantMs) + } + + // (e) Summary.HasSkips() == false. + if summary.HasSkips() { + t.Errorf("HasSkips() = true, want false; skips: %v", summary.Skips) + } + + // (e1) Summary.Large == 1 (records from large-message-sourced messages). + if got, want := summary.Large, 1; got != want { + t.Errorf("Large = %d, want %d", got, want) + } + + // (e2) Summary.Paged == 456 (records read from page files). + if got, want := summary.Paged, 456; got != want { + t.Errorf("Paged = %d, want %d", got, want) + } + + // (f) piping the emitted records through a real store.NewWriter + + // store.OpenReader round-trips byte-identically. + storePath := filepath.Join(t.TempDir(), "salvage.artx") + w, err := store.NewWriter(storePath) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + for _, r := range records { + if err := w.Append(r); err != nil { + t.Fatalf("Append: %v", err) + } + } + if err := w.Sync(); err != nil { + t.Fatalf("Sync: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + rd, err := store.OpenReader(storePath) + if err != nil { + t.Fatalf("OpenReader: %v", err) + } + defer rd.Close() + + var roundTripped []store.Record + for { + rec, _, err := rd.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Next: %v", err) + } + roundTripped = append(roundTripped, rec) + } + if len(roundTripped) != len(records) { + t.Fatalf("round-tripped %d records, want %d", len(roundTripped), len(records)) + } + for i := range records { + got, want := roundTripped[i], records[i] + if got.UUID != want.UUID || got.Queue != want.Queue || got.DrainedAt != want.DrainedAt || !bytes.Equal(got.AMQP, want.AMQP) { + t.Fatalf("record %d round-trip mismatch:\n got %+v\n want %+v", i, got, want) + } + } +} + +// --- corrupt journal record (finding C1: corruption must be reported and +// gate the exit code, not silently dropped) --- + +// TestSalvageCorruptJournalRecordReportedAsCorruption pins finding C1 at the +// journal-package level: a corrupted message-journal record must not be a +// silent data loss. Reviewer's proof was exactly this -- flipping one +// check-size byte in the fixture journal dropped a message with zero output +// about the corruption and exit 0. This test reuses file_test.go's own +// technique (scanCleanSpans + flipping the last byte of a record's trailing +// check-size int, TestReadJournalDirCorruptRecordResyncs) but drives it +// through the full Salvage pipeline rather than just ReadJournalDir, and +// asserts the corruption surfaces in Summary.Diags as a corruption-class +// (HasCorruption) entry. +func TestSalvageCorruptJournalRecordReportedAsCorruption(t *testing.T) { + dir := fixtureDir(t) + opts := fixtureSalvageOptions(t, dir) + + // Corrupt a fresh copy of just the message-journal sub-dir; Bindings/ + // LargeMessages/Paging stay pointed at the original, untouched fixture + // (Salvage only reads them). + corruptedJournalDir := t.TempDir() + entries, err := os.ReadDir(opts.Journal) + if err != nil { + t.Fatalf("read journal dir: %v", err) + } + for _, e := range entries { + data, err := os.ReadFile(filepath.Join(opts.Journal, e.Name())) + if err != nil { + t.Fatalf("read %s: %v", e.Name(), err) + } + if err := os.WriteFile(filepath.Join(corruptedJournalDir, e.Name()), data, 0o644); err != nil { + t.Fatalf("write %s: %v", e.Name(), err) + } + } + + targetPath := filepath.Join(corruptedJournalDir, "activemq-data-1.amq") + data, err := os.ReadFile(targetPath) + if err != nil { + t.Fatalf("read copied journal file: %v", err) + } + spans := scanCleanSpans(t, data) + if len(spans) < 2 { + t.Fatalf("need >= 2 records to corrupt one and keep survivors, got %d", len(spans)) + } + target := spans[1] + corruptOffset := target.end - 1 // last byte of the trailing check-size int + data[corruptOffset] ^= 0xFF + if err := os.WriteFile(targetPath, data, 0o644); err != nil { + t.Fatalf("write corrupted journal file: %v", err) + } + + opts.Journal = corruptedJournalDir + + summary, err := Salvage(opts, func(store.Record) error { return nil }) + if err != nil { + t.Fatalf("Salvage: %v (a corrupted record must resync past, not abort the run)", err) + } + + // The corruption must not be silently lost: it must show up as a Diags + // entry classified corruption-class (gates exit like a skip). + if !summary.HasCorruption() { + t.Errorf("HasCorruption() = false, want true; diags: %v", summary.Diags) + } + foundDiag := false + for _, d := range summary.Diags { + if strings.Contains(d, "message journal") && strings.Contains(d, "check-size mismatch") { + foundDiag = true + } + } + if !foundDiag { + t.Errorf("Diags missing a check-size-mismatch entry for the corrupted record: %v", summary.Diags) + } + + // The corrupted record is the fixture's message-id-38 ADD_REF: its only + // surviving queue ref, so losing it drops message 38 entirely (zero + // remaining refs, see message.go's DecodeMessages/decodeRefs). Verified + // empirically against this exact fixture: 511, one fewer than the + // clean-fixture baseline of 512 (TestSalvageFixtureGolden). Asserted as + // an exact count, not just "<= 512", since the whole point of this test + // is that the loss is real and precisely accounted for, not merely + // "not worse than before". + if got, want := summary.Total(), 511; got != want { + t.Errorf("Total() = %d, want %d (clean-fixture baseline 512 minus the one message whose only surviving ref was corrupted)", got, want) + } +} + +// --- fatal dir errors (spec §5: missing bindings/journal is fatal) --- + +func TestSalvageMissingBindingsDirFatal(t *testing.T) { + dir := fixtureDir(t) + opts := fixtureSalvageOptions(t, dir) + opts.Bindings = filepath.Join(dir, "does-not-exist") + + _, err := Salvage(opts, func(store.Record) error { return nil }) + if err == nil { + t.Fatal("want error for missing bindings dir, got nil") + } +} + +func TestSalvageMissingJournalDirFatal(t *testing.T) { + dir := fixtureDir(t) + opts := fixtureSalvageOptions(t, dir) + opts.Journal = filepath.Join(dir, "does-not-exist") + + _, err := Salvage(opts, func(store.Record) error { return nil }) + if err == nil { + t.Fatal("want error for missing journal dir, got nil") + } +} + +// --- non-fatal missing large-messages/paging dirs (spec §5: fine, warn only +// if referenced) --- + +func TestSalvageMissingLargeMessagesDirWarnsWhenReferenced(t *testing.T) { + dir := fixtureDir(t) + opts := fixtureSalvageOptions(t, dir) + opts.LargeMessages = filepath.Join(dir, "does-not-exist") + + summary, err := Salvage(opts, func(store.Record) error { return nil }) + if err != nil { + t.Fatalf("Salvage: %v", err) + } + foundWarn := false + for _, d := range summary.Diags { + if strings.Contains(d, "large-messages") { + foundWarn = true + } + } + if !foundWarn { + t.Errorf("Diags missing large-messages-dir-missing warning: %v", summary.Diags) + } + // The fixture's one large message can no longer be attached, so it must + // show up as a skip (missing-file), and HasSkips() must be true. + if !summary.HasSkips() { + t.Errorf("HasSkips() = false, want true (large message file unreachable): skips=%v", summary.Skips) + } +} + +func TestSalvageMissingPagingDirWarnsWhenReferenced(t *testing.T) { + dir := fixtureDir(t) + opts := fixtureSalvageOptions(t, dir) + opts.Paging = filepath.Join(dir, "does-not-exist") + + summary, err := Salvage(opts, func(store.Record) error { return nil }) + if err != nil { + t.Fatalf("Salvage: %v", err) + } + foundWarn := false + for _, d := range summary.Diags { + if strings.Contains(d, "paging") { + foundWarn = true + } + } + if !foundWarn { + t.Errorf("Diags missing paging-dir-missing warning: %v", summary.Diags) + } + // 456 of the 500 salvage.paged messages live only in page files; with + // the paging dir gone, salvage.paged should be short by exactly that + // many (only the 44 journal-resident ones survive). + if got, want := summary.PerQueue["salvage.paged"], 44; got != want { + t.Errorf("PerQueue[salvage.paged] = %d, want %d", got, want) + } +} + +// --- emit errors abort --- + +func TestSalvageEmitErrorAborts(t *testing.T) { + dir := fixtureDir(t) + opts := fixtureSalvageOptions(t, dir) + + wantErr := fmt.Errorf("sink closed") + n := 0 + _, err := Salvage(opts, func(store.Record) error { + n++ + return wantErr + }) + if err == nil { + t.Fatal("want error from aborted emit, got nil") + } + if n != 1 { + t.Errorf("emit called %d times, want exactly 1 (abort on first error)", n) + } +} + +// --- Summary helper methods --- + +func TestSummaryTotalAndHasSkips(t *testing.T) { + s := Summary{PerQueue: map[string]int{"a": 2, "b": 3}} + if s.Total() != 5 { + t.Errorf("Total() = %d, want 5", s.Total()) + } + if s.HasSkips() { + t.Error("HasSkips() = true, want false") + } + s.Skips = append(s.Skips, "something skipped: 1") + if !s.HasSkips() { + t.Error("HasSkips() = false, want true") + } +} diff --git a/internal/journal/testdata/artemis-2.42-core-data.tar.gz b/internal/journal/testdata/artemis-2.42-core-data.tar.gz new file mode 100644 index 0000000..0e3c6ce Binary files /dev/null and b/internal/journal/testdata/artemis-2.42-core-data.tar.gz differ diff --git a/internal/journal/testdata/artemis-2.42-data.tar.gz b/internal/journal/testdata/artemis-2.42-data.tar.gz new file mode 100644 index 0000000..4310493 Binary files /dev/null and b/internal/journal/testdata/artemis-2.42-data.tar.gz differ diff --git a/internal/journal/testdata/core-record-2.42.bin b/internal/journal/testdata/core-record-2.42.bin new file mode 100644 index 0000000..b492540 Binary files /dev/null and b/internal/journal/testdata/core-record-2.42.bin differ diff --git a/internal/journal/testdata/manifest.json b/internal/journal/testdata/manifest.json new file mode 100644 index 0000000..f1ddc72 --- /dev/null +++ b/internal/journal/testdata/manifest.json @@ -0,0 +1,2084 @@ +{ + "queues": { + "salvage.acked": [], + "salvage.large": [ + { + "bodySha256": "c16af104b017e07b3af46dfa8d2b3b63fcb149f469848810fe5f637412926d61", + "bodyLen": 307200 + } + ], + "salvage.paged": [ + { + "bodySha256": "424067dea2be2a8d763f59badacdfe77f3c464444952b4d23f78f0ed9fe9457d", + "bodyLen": 1024 + }, + { + "bodySha256": "379c2fbb842cf6080385d8415a8e593b145c8850f17e3271feff119492f9353c", + "bodyLen": 1024 + }, + { + "bodySha256": "b856ed96b68cb12335088748dac9eb8c0e8fce3d84582cb1d83b2b14d405e8ef", + "bodyLen": 1024 + }, + { + "bodySha256": "7e128912e61bf5175617c563f94d6e968ed36a347954363a6508cb455e22358b", + "bodyLen": 1024 + }, + { + "bodySha256": "e6cc25cb870d8a346e78031a6124b7bb549cdd54275684ce3b32f603b48753e0", + "bodyLen": 1024 + }, + { + "bodySha256": "d54095d70510d1d0f3c4a8cd3102fc952d0ce5d1391a33dfb82fd04ae610f4dd", + "bodyLen": 1024 + }, + { + "bodySha256": "b891ca3770183704e95bd6962d4ac35c44df8933c57f3e20cc7cb2a0051ab59e", + "bodyLen": 1024 + }, + { + "bodySha256": "e7a4ecdfcdae683e7efba338208486fa70bd121b811ddc825f4335055043fa45", + "bodyLen": 1024 + }, + { + "bodySha256": "cc9562508a8da8970088f512affc70d25863134321787e0d4faa4fb2ccc81c67", + "bodyLen": 1024 + }, + { + "bodySha256": "5894dd523ba990265bee16c2e71c7b1e73ac2fc4ea9549dfa5827892ec793636", + "bodyLen": 1024 + }, + { + "bodySha256": "f857cdbecceb7a4ca77d018515cf5247ebde13ca3e7edfae534cb2cc32bce9a4", + "bodyLen": 1024 + }, + { + "bodySha256": "6606c54e1d0a5f00cf430b1f695bd4d431b0862266e9d5da1e95241cea3bf83d", + "bodyLen": 1024 + }, + { + "bodySha256": "26608c2418dccec1c8c74cb5667f973dea08ac02101fc6584b6234c4a426f818", + "bodyLen": 1024 + }, + { + "bodySha256": "69f6c05386d523bc97bd5752d8e9950361b2c81d80de65033106b258de12e75a", + "bodyLen": 1024 + }, + { + "bodySha256": "3269704afc089b55e3508ace561f5be027290afd7010eaf2613e62e82654c3b4", + "bodyLen": 1024 + }, + { + "bodySha256": "ced5b0133a538e1986b415578ce45b69941fc6e5da82fff8455bd78cf6fa7384", + "bodyLen": 1024 + }, + { + "bodySha256": "b59cd01383f23ea701fe202df9640a16336faf7501d552045eae599c4fc640ad", + "bodyLen": 1024 + }, + { + "bodySha256": "b3a7da54f876f5747865b563594060ac730f08758e715b6addb361ac5f935bd9", + "bodyLen": 1024 + }, + { + "bodySha256": "1f0815ebb1ca4b40e8a7cd094915017c50bb72fcf103a0f140c21aa8a3c497bd", + "bodyLen": 1024 + }, + { + "bodySha256": "525414bfe62f2980df46ed70d45b6ad5dde76cdf444076345d182cdc66c5c851", + "bodyLen": 1024 + }, + { + "bodySha256": "23a99f7f13de935ef6e8c398a99436b44e8da89765a68426cd2dc4a3e63d62a8", + "bodyLen": 1024 + }, + { + "bodySha256": "273d9943220449bb7413fa95029474f691b687748ba632d12b05bdf7a03de955", + "bodyLen": 1024 + }, + { + "bodySha256": "57fc33d619e524e3b95ee2609637a6b210fc50dbc822b8441d453dc4ede7084a", + "bodyLen": 1024 + }, + { + "bodySha256": "81e0e36f80c77c4698701dfcb2c352d342e019e1c81835543866034efb170900", + "bodyLen": 1024 + }, + { + "bodySha256": "2e9a46163e0fd59c187e48705538a8d224056d5e66351be7549a55971abfe9ac", + "bodyLen": 1024 + }, + { + "bodySha256": "b12fff1e8fe64052a85ac1b9ddae9d17b1f592ea8aacf95da54a11fee9349026", + "bodyLen": 1024 + }, + { + "bodySha256": "c4dfa4efa9074486142cbea3e20964cac7d117772153f17e89dbf36023edc91c", + "bodyLen": 1024 + }, + { + "bodySha256": "402ea411feb6ceba916c2953b8b522608b5d3da031ec7c112d40e29867ffc3e2", + "bodyLen": 1024 + }, + { + "bodySha256": "85aa31cbcd2e030f587b4a96d7d5747d0b185b9d6d1b0f9e0498e6435735e78d", + "bodyLen": 1024 + }, + { + "bodySha256": "049c0939238d43ad6232aeef979edde2a881ff24960f8c6e01a684ce7d8e50ca", + "bodyLen": 1024 + }, + { + "bodySha256": "3e462bdca8c888f82e6c1786ad7d0e6598551d43da6f9025b06a327dd00dc717", + "bodyLen": 1024 + }, + { + "bodySha256": "41c3bfbea74dc262a6c1c49288329e25e0f40c2ddc52bc2bc9ab2ac2d45a9520", + "bodyLen": 1024 + }, + { + "bodySha256": "114d900c834e6e4e9de80416a0d49a82b18897766e6cf1657f16ce2f8a499fb6", + "bodyLen": 1024 + }, + { + "bodySha256": "9078d782e94cd42601b4b3b57649115a12bcd097fde625130d3db454451985a1", + "bodyLen": 1024 + }, + { + "bodySha256": "31e04ef88048b23401e048f9d65f23a31552209489d1b36622c10393b81c894a", + "bodyLen": 1024 + }, + { + "bodySha256": "94534f985cfe3d2ed42be0070fac868fa0b941e43a7137e3ac85fd83deb57520", + "bodyLen": 1024 + }, + { + "bodySha256": "e2dea40759c98b48c336a490ce717d969b754510360c31e7304a407d31b0833d", + "bodyLen": 1024 + }, + { + "bodySha256": "6f47adf6381b75930df4e812e0594b70927dea69e0c8011cdfe30b49a6d63172", + "bodyLen": 1024 + }, + { + "bodySha256": "a96308db8862e7bf88cba85630267cf120097751e009e5f93d75c1e90de90d84", + "bodyLen": 1024 + }, + { + "bodySha256": "c370315ca6fc960542bd57594590d812263ea8001b7e811aae11ae955f0a838b", + "bodyLen": 1024 + }, + { + "bodySha256": "2f91a7d1f9aae74b11acf4d5636a13036c8d1f833bc887df33a9c2e5b3bec392", + "bodyLen": 1024 + }, + { + "bodySha256": "dfc0b4e7b02c0349a4b71db7e59b0d5a1a24e6d6182bc05ec16ac69fbeac4555", + "bodyLen": 1024 + }, + { + "bodySha256": "f00248e4fec94f20d160afb4b08196679788016833360a431749b21d970a20ce", + "bodyLen": 1024 + }, + { + "bodySha256": "38584045170517c7ae82ea69b49cdd8f5cf56368f9d8c73c597b97efd38122ae", + "bodyLen": 1024 + }, + { + "bodySha256": "1bba32a01893165e225bb3035f70df089e8b9cd8107a4d52a1861b2a5c38531f", + "bodyLen": 1024 + }, + { + "bodySha256": "6ad01a6d077bdba0310f835f7373599c1fd2ebd2d11979f62c7759c8d4a11156", + "bodyLen": 1024 + }, + { + "bodySha256": "000e452594849a456a85f7037871e2b2584d88b8209037c3384b176437be1189", + "bodyLen": 1024 + }, + { + "bodySha256": "967b6d05de73ff3b62eec36553c5a641babc1b0edb9bc644328aab047212fcb8", + "bodyLen": 1024 + }, + { + "bodySha256": "4d7fe1737dcbac425944daf289ce245df14d6d797bf7f2821117833b92eade5c", + "bodyLen": 1024 + }, + { + "bodySha256": "097531eb9b036ddedcfe4d70259a7d956b56a08c153d531e0aeef12a1814e23d", + "bodyLen": 1024 + }, + { + "bodySha256": "522bb86d8ff43b01d34d78ba6c2b2c595e181031055c27619731c1932909a2c2", + "bodyLen": 1024 + }, + { + "bodySha256": "1223ab25d4c68a9aa188a09d6b85756685ea4bc7277ff60ef8c6329d121f1b77", + "bodyLen": 1024 + }, + { + "bodySha256": "5995ade58d2f2cb09eb01ce704fd9863eb2750717f353fbf1f6ff8fe29bb5922", + "bodyLen": 1024 + }, + { + "bodySha256": "12bd56a5ab065e73bf8fffef2f6b477f3a048b6dac1c96fc0b37d16f3479856c", + "bodyLen": 1024 + }, + { + "bodySha256": "0d28691029bf38b52dd15461af20bc1f8ab1ade8b35a9330a9637823917f1127", + "bodyLen": 1024 + }, + { + "bodySha256": "cf9b1171dd3acc2ed7a86630d8c113ec3ad5e1504f8c8a78ab16dda836f4e31f", + "bodyLen": 1024 + }, + { + "bodySha256": "1f37cb1db3579288622ffcdaebdde4b7b6379ae20e3d94223077ca96fd4884c8", + "bodyLen": 1024 + }, + { + "bodySha256": "6d01dc734ff674bdb8d3742d00f0f5738136982e021c7f2f9537b1e5414aa3bd", + "bodyLen": 1024 + }, + { + "bodySha256": "b857be562ed34b709243198c7f75ec50e9117e0fc699aaae4bf9186049d14098", + "bodyLen": 1024 + }, + { + "bodySha256": "9841c992b423e08eeec7a0c7231f1b33591fe67d1580ddb2925c9cdb42fa9210", + "bodyLen": 1024 + }, + { + "bodySha256": "c5376ef9394c68f55fec2506db2529209a68760b6d208e43cd990d4058a3eb10", + "bodyLen": 1024 + }, + { + "bodySha256": "50f922b0a8de97feb3cda56bdc2714770d40fbcb1eac4692eb3e0b9e86e10552", + "bodyLen": 1024 + }, + { + "bodySha256": "3d60fd904905e15cb23bef79df16a22f41dd2cb471f6cbc30de2c0064ad6dc22", + "bodyLen": 1024 + }, + { + "bodySha256": "8c3deee647bd2dd20c3fd54229dbd78a40793b13b82ed949347571d7e099aa9f", + "bodyLen": 1024 + }, + { + "bodySha256": "8bc252dfe3242673ca0cfa2a17f7f034bc689c0402b1c72f900e669536f8feb3", + "bodyLen": 1024 + }, + { + "bodySha256": "6d71079a3c7aff1221019284235518b2b64542c79bb22e665bc997bcc6274cfe", + "bodyLen": 1024 + }, + { + "bodySha256": "bf549625810dbfa80a5bfc299d88fa49f2c0b589ca62075ad55b6b3e97cbe0db", + "bodyLen": 1024 + }, + { + "bodySha256": "e8796b0c251d50d166b97d67b793120f6a17347b5c25f74c99de690114c44994", + "bodyLen": 1024 + }, + { + "bodySha256": "708d5c9c9981a7728532499dc8363fb158012a3c69dbf1a7ea4ff1b1f8e55748", + "bodyLen": 1024 + }, + { + "bodySha256": "64b369396c93438f66ab23c311318c04d7b9e9ab5eb5fcbca33082ef1bc4917f", + "bodyLen": 1024 + }, + { + "bodySha256": "eb50a82fd10202c6f09e6d1c8240cb0e71bf20f326c5d69df493cebdbcea3830", + "bodyLen": 1024 + }, + { + "bodySha256": "9c7fca799d3e6b365200852216384334722a9d1f455fd4af867378da95e90f3c", + "bodyLen": 1024 + }, + { + "bodySha256": "a5b57f6effdccf632cdfcff1ce3b61c9828b896244037909e98e44cbcc374c86", + "bodyLen": 1024 + }, + { + "bodySha256": "328120ef53cc11605ff2f2655592007a28f2d664d6c62da4ec1d2cdb55a67bfa", + "bodyLen": 1024 + }, + { + "bodySha256": "ca726389784f478b086e5582ec011fff8f5cb7c6f2641f8b23350a033d3a542a", + "bodyLen": 1024 + }, + { + "bodySha256": "3c6be0c2c30c1faaf7b65adf7c53c3056e5b33e3ec848a69fb9792f0d274bb32", + "bodyLen": 1024 + }, + { + "bodySha256": "8fc68013b07301a676142042830dcf2009e36604fe3889f59e4e3f8a8fe5ddee", + "bodyLen": 1024 + }, + { + "bodySha256": "12fb3504111759850bef28d8970bbdddb92281c161f42099154b05e5e94cb791", + "bodyLen": 1024 + }, + { + "bodySha256": "ee43d1e69dbe1e7fcf0841be1d09c96e2d2b1933c1fe5789998ed606fd43f86e", + "bodyLen": 1024 + }, + { + "bodySha256": "b95a710d11aa7218568a0210a7d6186bb795699e3f5c259ed23619760a4a9ed6", + "bodyLen": 1024 + }, + { + "bodySha256": "b0a5ca724129b9a198293908bd8956a54d649c1f3040347ce41992e0ed70f4ee", + "bodyLen": 1024 + }, + { + "bodySha256": "40803ce91948172443a6628fad3deeae08c472c2bb5880df339c0c363ce6b9ae", + "bodyLen": 1024 + }, + { + "bodySha256": "ec449ee09a053c63434059d2d2d7db11eb346a0de56213c75c0d9e668b18765b", + "bodyLen": 1024 + }, + { + "bodySha256": "fdaf8a2f107ab5d7cf1dd802e0c1971f14e5c3b564ec665b52029d14c0d8cd9c", + "bodyLen": 1024 + }, + { + "bodySha256": "41dcfd99fec441a11a7d149effe00b6b9a6973e7d0dcd4b26dd59e3e02be0f40", + "bodyLen": 1024 + }, + { + "bodySha256": "d00c97a25d825199043d1d51252a75ffdfe4664cf1d1a0bde6c561bf6b9c12e2", + "bodyLen": 1024 + }, + { + "bodySha256": "1c4a1d886c742e8fce0e6c8b66a8c2750702581021d541d10c9e3bf99f66e4b1", + "bodyLen": 1024 + }, + { + "bodySha256": "aae6920de4eaa6a0044f790f1f160f6ffc25045a729e2942731b212d2b8c476d", + "bodyLen": 1024 + }, + { + "bodySha256": "e19c40d147551834f2464136eff61bf0035dd322a1f5b883d6852aeb4e6a956c", + "bodyLen": 1024 + }, + { + "bodySha256": "ce8fda32f3bb87968ef2c92dcdc52e5071cb32684118858b3410c5630b2323dd", + "bodyLen": 1024 + }, + { + "bodySha256": "ad475a14b789f82be2355ac0385898e0c86f569586b2e2f5c723bf88611b5483", + "bodyLen": 1024 + }, + { + "bodySha256": "e175ad958f907e5d14f0907863e25787278aaefdebe804a1de505d9dd646fd4d", + "bodyLen": 1024 + }, + { + "bodySha256": "207fd31a7869af3360791dca49958d8e52bd7aac5c15601dc7e5ab39be4156de", + "bodyLen": 1024 + }, + { + "bodySha256": "8b26e10f6c4b1caf6ae4c4ca10bf251997429834b1a1f17bae7bf690b5c904e7", + "bodyLen": 1024 + }, + { + "bodySha256": "1419c204666fdbcf4ff9066cb700e1f5f3f971f3c4b04d021042fe3c3039e991", + "bodyLen": 1024 + }, + { + "bodySha256": "a8d498759a04330c8252a33c97d47a9c0187e8137a0d7cfcdafe9baf41c2c62e", + "bodyLen": 1024 + }, + { + "bodySha256": "5de520f65d09267a9c2895aef4c2b171030581c1f883a37a01d409f4dec0a78f", + "bodyLen": 1024 + }, + { + "bodySha256": "bdcc5cfc16be5bee265945a1f8573b7f0fbf9cf3d3a9647f5eff948ef10378b8", + "bodyLen": 1024 + }, + { + "bodySha256": "01967d25b67b72dc32a57bbc5ce0306153930c82c516d80b75ee028a31599456", + "bodyLen": 1024 + }, + { + "bodySha256": "23c19699452e55d38e23455cdcbc889b02c7efd6e5bdb0d1e668144d8d7250d5", + "bodyLen": 1024 + }, + { + "bodySha256": "c16ea8734646abe32e773d361211b340dd761af4842369844ec29b1bdbc4350d", + "bodyLen": 1024 + }, + { + "bodySha256": "2a11192a7fe999e5df599e1db8a11627ad8f69a034f95bc8867db62e2593b6c1", + "bodyLen": 1024 + }, + { + "bodySha256": "ad56c1bba9a6f085e8545f8e6da7e6888e8312962ee707d9bc9d9dba3408613e", + "bodyLen": 1024 + }, + { + "bodySha256": "d80f45131465db361eb736d15e9aeb2f4945d7a927f2ce68a2c0ea05e26f1d42", + "bodyLen": 1024 + }, + { + "bodySha256": "0d8cca800fb3ad2f2e82184f845075b339995148b531ce0b5fe9c075be6d3dc1", + "bodyLen": 1024 + }, + { + "bodySha256": "caf581124857e4e287a961254004dd8ebbaba1327c191a1dc7e8f2738b4caf0a", + "bodyLen": 1024 + }, + { + "bodySha256": "77db553466f7c66f9ac2fade891fbc1f8e0d47ee1c00abca1b7f11b8d9270b80", + "bodyLen": 1024 + }, + { + "bodySha256": "975780be056a283e097cae7b1fd9191942b7a683044d88dcb25ec6b765a61e3e", + "bodyLen": 1024 + }, + { + "bodySha256": "d7eb2b26d082a0393c4a3782825aebd781f7a6fb955ed65dcb00eb4855504ab9", + "bodyLen": 1024 + }, + { + "bodySha256": "f2ec954e41d4674e23f11e354c084481cfa27653d40c85f326ad2359e351f9bf", + "bodyLen": 1024 + }, + { + "bodySha256": "cc8d3bb6a741ce8818ed8e93ece17fb4e5a984433d196b79a0b05a407d3850bc", + "bodyLen": 1024 + }, + { + "bodySha256": "7ae078c084e7283cb0c1a5a4f4ea68f250232fa1ebf20782309a052fa170af9a", + "bodyLen": 1024 + }, + { + "bodySha256": "3d9e7c9f21dd13b45df76f4559ffc5d4fbd5481911e295116b888ef2da7fa062", + "bodyLen": 1024 + }, + { + "bodySha256": "cc0839559caaaa02c5b4d53949169202e64eb257b92ed2946147039da57adf47", + "bodyLen": 1024 + }, + { + "bodySha256": "4a2e4f2ac28154f8cc3f13c39138bbab7dbebab1138d54681d42de9a3eb87f93", + "bodyLen": 1024 + }, + { + "bodySha256": "4c0f463afec3c8148602127fea84285ecbd4a5f942bdf02479576c256a76ed12", + "bodyLen": 1024 + }, + { + "bodySha256": "fe24dde36ca1df30db7e0adaa78b095932adebea9006fd3f8625af0deee2934f", + "bodyLen": 1024 + }, + { + "bodySha256": "d49d40205e6bf1957bb7ffb5e078505238a02b251666c2281090b199e955edbd", + "bodyLen": 1024 + }, + { + "bodySha256": "9c5a530ab3e9cea9e6b311b56765ad304d8186958df5ec2bffc11770150cfaa1", + "bodyLen": 1024 + }, + { + "bodySha256": "5fca122925975bf9b42248b106c3f846534b3062bb94fd091967153c55ec1495", + "bodyLen": 1024 + }, + { + "bodySha256": "09817c6f969711e92a9674558dd1e6fb2d5754d03d989fa6164bae2c44b76638", + "bodyLen": 1024 + }, + { + "bodySha256": "c36c8ed8efb9403158ad45d8fc2e71dcdc67d3423bcddc7adaf9e4bd72dde1ba", + "bodyLen": 1024 + }, + { + "bodySha256": "bcec6b77fc5b7c36afdc9ed9663649315432fc0edda4fec3020f1c3abd9b5957", + "bodyLen": 1024 + }, + { + "bodySha256": "f5c4323dc084e27f4d3b3a562f6bc1cad72e0467e4c2e41bbe22911f5f0e8bb2", + "bodyLen": 1024 + }, + { + "bodySha256": "d070bf59338c83369bd03a5b078c81473a1c8742d97f73f6b1694cc2adb13dfd", + "bodyLen": 1024 + }, + { + "bodySha256": "d422b842cc3dd74a10e2f8765e4c60436c53ce5aae0f162f8b70a79caf2d96af", + "bodyLen": 1024 + }, + { + "bodySha256": "5f43f181e3bc2cec547b963d6061caa830ce7e8be3c2ebd06f471d237365b97e", + "bodyLen": 1024 + }, + { + "bodySha256": "ee708d584cb77a9ea5d800c73f987ea52ffc54a43e0c5550c6fb95d4fc3366a8", + "bodyLen": 1024 + }, + { + "bodySha256": "a9530a7295c9d2c6012ca16fe5daeb2024d6fd1a1f55ec3e90d3158cb52e4a06", + "bodyLen": 1024 + }, + { + "bodySha256": "5c2edb0db5ebc5adb1a70a729affb5f2439beaeb24346b209218270db0d144ff", + "bodyLen": 1024 + }, + { + "bodySha256": "8842b783356ff28875d56203bf8a5ba6b92881de2ea1694f8392018a5838de28", + "bodyLen": 1024 + }, + { + "bodySha256": "61d3cdb51af956ca5ce1ac6f9737dce2d3231f093afdab75b1c5b2b4cf8ecd70", + "bodyLen": 1024 + }, + { + "bodySha256": "ed99fba9284748a30c071d8fd1cc138d6c6b61bb3572fdbaa7158529a951d274", + "bodyLen": 1024 + }, + { + "bodySha256": "92ef919b4f8b7bc74afd631119e2e12f0ff4acdcc182ce6b938d60abaf03e71b", + "bodyLen": 1024 + }, + { + "bodySha256": "61f7b8f1c4e7e1a45dc492582e51af3f4daf28d93807768d8c711bd82937bb01", + "bodyLen": 1024 + }, + { + "bodySha256": "2b5d3bcc111116839ddc4fcf4cae0100cd6f3bd14b8f07e899d6355af910c7e6", + "bodyLen": 1024 + }, + { + "bodySha256": "63d94bd4bc0a42208c008ef3be6fd094b050ae59ebb109e08ac6ef158523c508", + "bodyLen": 1024 + }, + { + "bodySha256": "0de14789f082ac00f2575caae36d3ed0bf52e10388260182e940812f78713d38", + "bodyLen": 1024 + }, + { + "bodySha256": "d596bca612363fa19d524d5ce16c2c5ca0d7f10b3ffff827816d1431d1e554e2", + "bodyLen": 1024 + }, + { + "bodySha256": "7e06876568e262729dd8cb238243d7fd0a80055f39227b608b6423385c2d82ec", + "bodyLen": 1024 + }, + { + "bodySha256": "57dd3a5837eef4fc7063b8fbef394bbdb7f8c0a5c5484182c441b7709ccb0d11", + "bodyLen": 1024 + }, + { + "bodySha256": "a8eece768ba56994069e88623b07ddeafa65298481e8d2ef2648f25d26ace428", + "bodyLen": 1024 + }, + { + "bodySha256": "5653dc6e28fbf661ad1fd2093c33937c1c41570b4acf2ae9fa0c959b3de0d355", + "bodyLen": 1024 + }, + { + "bodySha256": "512c2b28ef026ea524e90d36468d7dfba5883da4a816080f275e29f4ed91d1bc", + "bodyLen": 1024 + }, + { + "bodySha256": "94b0f9f22d6063ddb1a137fe05075059c0e0aa89a642a0f8e93d06c95566b1fe", + "bodyLen": 1024 + }, + { + "bodySha256": "1ea2eddce99f0ea541e9e780372235934f7b7d2d924de1d4170d1278b9dca2ae", + "bodyLen": 1024 + }, + { + "bodySha256": "a8aeb8c55bc35666318d49c453510959a59b72fefee487206f8d052604eb8d73", + "bodyLen": 1024 + }, + { + "bodySha256": "8f8a1da3de4f9c0c1732238a5e955bf60b276ff649573699e2a7c6a2dc211aee", + "bodyLen": 1024 + }, + { + "bodySha256": "36a3a5daf061564c557c290a9982bc957df42775dba73608e772a8858a4637b7", + "bodyLen": 1024 + }, + { + "bodySha256": "b4e2705940a8922c2984eba816a5179d8fc4a988b2ae4e9292b8ce77d2f01673", + "bodyLen": 1024 + }, + { + "bodySha256": "d5b97e57b0eec1419f24285ef91dfdd4d774587decdf90f2c1fca6d177041277", + "bodyLen": 1024 + }, + { + "bodySha256": "ed1c979e8b83de326133ed9cbb0aa585ded295fdfc9bc2e2d4e9fe01969e4b70", + "bodyLen": 1024 + }, + { + "bodySha256": "3f0aa5c5a7ae327ecb8222d5df0c095961c652b904d4c1e75d9aeb410577fa29", + "bodyLen": 1024 + }, + { + "bodySha256": "9bab96bc4bb3b05f60352beae1e0912683669a4da7a024f08be781da3114e60e", + "bodyLen": 1024 + }, + { + "bodySha256": "e714bbe6f845cec951153e017b0714cbcff331556ef4931e24b3382a5841dc05", + "bodyLen": 1024 + }, + { + "bodySha256": "3751e2415cb5a629e35a591b4099b73f1c7a0ea735a55dbd5b6e579f8e6fd341", + "bodyLen": 1024 + }, + { + "bodySha256": "17a6d61b907eb6c26776893657ccb201e05aeb8199470d681d6a45de4ff190f2", + "bodyLen": 1024 + }, + { + "bodySha256": "74a82ea425b599812f48c9c54d2cd3212549ff05c56cbbca2f4ebd9e75ce36d5", + "bodyLen": 1024 + }, + { + "bodySha256": "d0ec9d1819a23d061a665b28d0992d232e15103982fd4bc240912a206bf722f4", + "bodyLen": 1024 + }, + { + "bodySha256": "0db4550af3d28959481c0f46fc72b4844b2c574d3f5feea5f9595cc8de7d65a7", + "bodyLen": 1024 + }, + { + "bodySha256": "9f53fb56142f54d7e766262f99e6a7843a7d3b7a3bb18a68c845fe9abb51194b", + "bodyLen": 1024 + }, + { + "bodySha256": "6b5ec7ba8b90239e218fd28cfe5eeaf2ff62847759fd27680f41eba854dfaa12", + "bodyLen": 1024 + }, + { + "bodySha256": "0c6360e146984fd11143e0166ca49b5e83bd9229efa147c8f936deb2f8aacd06", + "bodyLen": 1024 + }, + { + "bodySha256": "e0a0213cd5c588b57762392260ce28213216db9f4a668ae20ea0525e8172a48d", + "bodyLen": 1024 + }, + { + "bodySha256": "e4bda7fb431717fa63a247a55512a34a989a5ee9493e4bbf58bdc4af82578cdd", + "bodyLen": 1024 + }, + { + "bodySha256": "d306ba0ad185be79dc79bb32ce9f592177640607050294898970bd2dce77c00c", + "bodyLen": 1024 + }, + { + "bodySha256": "7be33761b2cac71a7be75706eb535867c0f4a7a8facf0dadc748eed918195f37", + "bodyLen": 1024 + }, + { + "bodySha256": "51c7e03c5e9158941f21c8158415cf69f9745bc9e196db52c23f14a673349b80", + "bodyLen": 1024 + }, + { + "bodySha256": "91a637c04c386b19d03d8d318c69acbd2a45b02c9d903ed739163e02d8e89f91", + "bodyLen": 1024 + }, + { + "bodySha256": "e2736d74fe067d068ca5347ef2e4ebc6ea7fa667faf698781c8eb186f1599687", + "bodyLen": 1024 + }, + { + "bodySha256": "78952bde3a4f3a76484e6ccc84883b091cbe778ec790573e177e6c69d9808b96", + "bodyLen": 1024 + }, + { + "bodySha256": "f5b440074d695879b387107fef865d304a75ec7bca684c46e379acddbbae3035", + "bodyLen": 1024 + }, + { + "bodySha256": "1978a7d9cf58285a51aedbbb9dd88b09e0e2de9cbbee380e878ff61d35e8cbae", + "bodyLen": 1024 + }, + { + "bodySha256": "6a4a36bdf29823ba05d8e210db09f01fd4236b66b51f13d71f3490c2fab92a31", + "bodyLen": 1024 + }, + { + "bodySha256": "6425989ebd5da14fef97139f2ec657b0e6b6394121b2043c55b126d791549954", + "bodyLen": 1024 + }, + { + "bodySha256": "62023fecc78299a9b9a98dc9ef8064f89c516f703957d45f5ab7659f00656209", + "bodyLen": 1024 + }, + { + "bodySha256": "8f1f766fa0e0512768e12818b531f75e814d663bfaa502e9a3896f4facbe9bef", + "bodyLen": 1024 + }, + { + "bodySha256": "854d8a1c82cfcf7685de08e5ec3218ec15d9bdf055461276d7851cf874b2babd", + "bodyLen": 1024 + }, + { + "bodySha256": "fb71ba8c9023d15fb82413c17f3eae3c37b883d85dfffc5c7ccecab7d823d3c8", + "bodyLen": 1024 + }, + { + "bodySha256": "73dc6ff335e7f27c2cf60523eb2a24b3e79856f5649c70c486e7411421677b49", + "bodyLen": 1024 + }, + { + "bodySha256": "ff51004a19684a7fc969e70a638bdda1c9766047330bd943ad9ece6ed0401acf", + "bodyLen": 1024 + }, + { + "bodySha256": "6a4fa3ca61ced9203f5fbed241841fdef60506c7245f11190edde6828994482e", + "bodyLen": 1024 + }, + { + "bodySha256": "996566d67e5cbba68feeb875f4c6947860980652cab720c45d780fae3be7c007", + "bodyLen": 1024 + }, + { + "bodySha256": "57256e3cc9100f253bce599a385d5a0e0c3bbd8ee309a13f435a8a8d247cc5b2", + "bodyLen": 1024 + }, + { + "bodySha256": "d9f1a506bff85442505646cbcfcd420a841d3329f83708f32aa50295323bd68b", + "bodyLen": 1024 + }, + { + "bodySha256": "17dc7713afc9c5ed3136a37eeec1d33d4b512af0f4251b827374250c5eaa8633", + "bodyLen": 1024 + }, + { + "bodySha256": "6bba254ce54ae518c111358c591f9cae066f3e8e86b1ce2032fcf84c65491e7a", + "bodyLen": 1024 + }, + { + "bodySha256": "00b4562e268e7dd22b53a2455c714e2fb6b491e2d3de7b675da4e9efb58d8b2f", + "bodyLen": 1024 + }, + { + "bodySha256": "e26ac599b2e69f1167b8b0bdc1045d5f8115c4d381496dfd7cdeb4a413c8879e", + "bodyLen": 1024 + }, + { + "bodySha256": "da4a1cf503225712e7efb6050319673d544a2a02023df269c2dcb7ff63f377ee", + "bodyLen": 1024 + }, + { + "bodySha256": "ff1f727d7d7dcbfd6c6210a59c24dc12276ceb2198421cf9eef08ba7ea5eab08", + "bodyLen": 1024 + }, + { + "bodySha256": "641c0dca692198c55a9df7f12534e70df7a70c1c86af62db5fa30dc78a0df9b5", + "bodyLen": 1024 + }, + { + "bodySha256": "382aef66345a97835cb38121b2fe5b8d909582e7431c1cef4a95ff633c4293a4", + "bodyLen": 1024 + }, + { + "bodySha256": "e8220b021f7f5fb17e40397aa213aa77df657d30f18b3428f1248a587d23b9b3", + "bodyLen": 1024 + }, + { + "bodySha256": "088eb18375bbc9e11d3b57f99c87a5cbb8f9822d3f8f99d7eb6d9ac4db90f75f", + "bodyLen": 1024 + }, + { + "bodySha256": "f5020c80eb5eed827649a210f8cbdf217d5e24a3e127012afbe97e5aa0f7f627", + "bodyLen": 1024 + }, + { + "bodySha256": "3d688c269e79e2d01fb1496fed171f62a1b7443577873896ed46eaa47ad7f101", + "bodyLen": 1024 + }, + { + "bodySha256": "d50fdcb7cc547d1f4fc22f1bbf044a7084a95a020e9976ad2ac2127c50640ce9", + "bodyLen": 1024 + }, + { + "bodySha256": "c7096be7498a5dca2e69d99dd96885ccee5311c379a1701ea25b10fbb44ed189", + "bodyLen": 1024 + }, + { + "bodySha256": "6e0fe0fbe240ec0639d37e4eae03cfc8204693ba98288aa9638370b265f5e722", + "bodyLen": 1024 + }, + { + "bodySha256": "23652754bdca3fb54a4e54572fcc89b5abae0b1bcec99bc28f852442dd72166b", + "bodyLen": 1024 + }, + { + "bodySha256": "ea614616d1c6dd2e8ce2e2bf67045aad914630ba01d433ed25ff2d24ddebf2e5", + "bodyLen": 1024 + }, + { + "bodySha256": "3645e4d8447e21184156aea33dadf3a772f37d9d2c958795c8e35d53c92f3c99", + "bodyLen": 1024 + }, + { + "bodySha256": "7a280f0354850ccb492762056221474493efb61821c51821b9eab05f96bb2c40", + "bodyLen": 1024 + }, + { + "bodySha256": "bea566cb471903fd027170f76faba6b358630937034331b26d78aa52d6bb37d4", + "bodyLen": 1024 + }, + { + "bodySha256": "b882775492f41fbed9fe0c99781b0ad122860510044c2dbee4b7b22ae685968f", + "bodyLen": 1024 + }, + { + "bodySha256": "b9c37661f6e7a6fd007157b00a3685bd1d7600d9446fdf572b805695e47ccdc0", + "bodyLen": 1024 + }, + { + "bodySha256": "283e7d0adece129ac4ed04da70e6fd2e36a22bbb3e438bdb5293b02434ab97c1", + "bodyLen": 1024 + }, + { + "bodySha256": "4037e1ccc57a5000b735ff0159aba0fa47f04528c75fcbc78bcf83abe3933d03", + "bodyLen": 1024 + }, + { + "bodySha256": "71492cb027de20ef1a72b48aa4e62ad5d06317ffb40a9aad630990aeed80d089", + "bodyLen": 1024 + }, + { + "bodySha256": "df6d7857391253483e90fbb76ddb6189966e3acc235352502cb2154839459baf", + "bodyLen": 1024 + }, + { + "bodySha256": "5a7a8e7a35e2d68a64cb203cfde91f52d3750339b3603cb208e9f26854f112d7", + "bodyLen": 1024 + }, + { + "bodySha256": "864b593a61b78cc03c2cda95443861a61559640f8fbc6c2b4308f520f21d7f29", + "bodyLen": 1024 + }, + { + "bodySha256": "908fba81125be1c939b34b33b35a456e08a57a1105f3d01357d258bf907151fb", + "bodyLen": 1024 + }, + { + "bodySha256": "6295b75b96fea11cd076eef15d2d5df2f2759bd4dc689177372eafb6c9fc86a1", + "bodyLen": 1024 + }, + { + "bodySha256": "541e62e6cc1cbcc44caa80b12ac9a2c842db63d12ebdf4642fa1ed3e91701227", + "bodyLen": 1024 + }, + { + "bodySha256": "acbadf06aa9bfd1aa674fe39954cc4b28c13764f312489a10e8e6905d89b8b80", + "bodyLen": 1024 + }, + { + "bodySha256": "7db3cc96471bc81d34d9c4d057f4d412aa92603225b4a4d34a5bfe01d2ef56ba", + "bodyLen": 1024 + }, + { + "bodySha256": "00a3294ea92f69bd0025da8ee7244aee998d124584ae3515d518a98247622576", + "bodyLen": 1024 + }, + { + "bodySha256": "04f9b097642822d1635c4c455f14df535797eedcfafb993b69e9e770630bc2a3", + "bodyLen": 1024 + }, + { + "bodySha256": "93f07c95eed53d422920ced1ee08d66be48021b36aaaa24d1a2bf9bbf78950ff", + "bodyLen": 1024 + }, + { + "bodySha256": "dbc4043d8f93909b4384f3894a22a5b73bd914bc3aeff66e4176fd3fba20879f", + "bodyLen": 1024 + }, + { + "bodySha256": "6270974c9768ddb8956d239103df422e768379b715399ba88fd90d9c7cf4e250", + "bodyLen": 1024 + }, + { + "bodySha256": "d22b954bbf6022f902a2111075a7dd4b15168e426fa66ef06d5d0d6fc81a57fb", + "bodyLen": 1024 + }, + { + "bodySha256": "8eb7a44ac26d4d905819e940be087d31f32abcc6b04c22df356d3f4ba936fa87", + "bodyLen": 1024 + }, + { + "bodySha256": "b9ca2d994b8cdf65fbe5fbef44c69d0b34b054a9d10ac0ee8c8b653948ff4347", + "bodyLen": 1024 + }, + { + "bodySha256": "990731f71b1f8da616f57ec4de5ee06c3e584647628cb658a4883cbca0509628", + "bodyLen": 1024 + }, + { + "bodySha256": "7dc1bc03006125eb5788dc8adb6b665af47280716d2ed8f3749396e943c5af0e", + "bodyLen": 1024 + }, + { + "bodySha256": "ab194120bdccf051dc9152d51a9a9954132e5997bf2619194fa151984b6853f4", + "bodyLen": 1024 + }, + { + "bodySha256": "b169caaa44d4f1c1f57d31817b906dc9b1edb9e7ae5732f066a413449a515988", + "bodyLen": 1024 + }, + { + "bodySha256": "d1e638ff8cd783f94ef5c0c1f77be4f7e862213e1e4e93090cf8fdb3dc71c7fb", + "bodyLen": 1024 + }, + { + "bodySha256": "b73552b24bdfd85f46692c4bbf8c959272b109760a7e5578fcc21e84f2a7521a", + "bodyLen": 1024 + }, + { + "bodySha256": "cfb6888307f6068101131fc57ad4bf0cd0ba4c571b8f8af5cee916d07024c2db", + "bodyLen": 1024 + }, + { + "bodySha256": "db6acb399445c91cc758a2f855ee7c8eaa11d0e4be682446024ac139b705471a", + "bodyLen": 1024 + }, + { + "bodySha256": "bb3ecd8ea0f5e8e0b32a8a5eeab8078e1b198c700c7fc3a9674d099f27f79aa1", + "bodyLen": 1024 + }, + { + "bodySha256": "1f74a79fe2d343162378eb3f2c3baf90940236bbc90380c88b7bb77706324b7b", + "bodyLen": 1024 + }, + { + "bodySha256": "fcbc89752becc781bb26561995afc4277201945ae6c31d0ffd7a352855517a94", + "bodyLen": 1024 + }, + { + "bodySha256": "94119b414f0298bc94060fd5daf994ef7843cb5765947a48de0a9bf04b28c536", + "bodyLen": 1024 + }, + { + "bodySha256": "b0ff7f3309f3141d4a156477e504969ba987bc5f64290cfba3998439fcdae8ae", + "bodyLen": 1024 + }, + { + "bodySha256": "f826c225b90b0e5c7b752ee135a6f62cc88be598f865ac50526d9d66a522621a", + "bodyLen": 1024 + }, + { + "bodySha256": "cfede605c7fa8920bb329f76001b6e5e1e77f48f9022befae23a76fd8109ad65", + "bodyLen": 1024 + }, + { + "bodySha256": "c61def7c66e0230d5b827d0f4c7114f264ce4cf966ae94e5dac4b0cb9cb6d8a6", + "bodyLen": 1024 + }, + { + "bodySha256": "571f0bb4eb99aeb21caf4927a419f95e1e4608f0de86358043565bcfa9ce2a47", + "bodyLen": 1024 + }, + { + "bodySha256": "7dfb9c1e7dd3518e30cf2919f6b0fd5c91c934ace414b2e5733f47b6c3c21771", + "bodyLen": 1024 + }, + { + "bodySha256": "2575c42db8e43641c68354bdcd73d3c60f49a602514738ec1780f56a30697d5f", + "bodyLen": 1024 + }, + { + "bodySha256": "1a26b1caf54a57b1d0738341b91a736c771383f96ed5fdc326fae2387baf7ad9", + "bodyLen": 1024 + }, + { + "bodySha256": "6aef3665055b90d4bc45f5b3d476f979c1d8f547da5898e5b88b6e6034eba9e6", + "bodyLen": 1024 + }, + { + "bodySha256": "faacd5034ef8b875e914716e09a4e252e5d2a5614052934409a4e83c9219a8b6", + "bodyLen": 1024 + }, + { + "bodySha256": "3cb8aaa375ee37010280a649dd71dc5ae5edc3c2cf89af28d8072adb72bc5908", + "bodyLen": 1024 + }, + { + "bodySha256": "2b6d63dc1e0cd76b16aed2989f4c26b0b9c677faa906d68dfac32a162532c191", + "bodyLen": 1024 + }, + { + "bodySha256": "24303f5983be6825df8ca5b29f612c81de17b692508dd6bf1817392b39e1f0d3", + "bodyLen": 1024 + }, + { + "bodySha256": "43eb4a340505ab4e41b656a1435f03e69a70184abb058577ceab4cd53dd3c2b2", + "bodyLen": 1024 + }, + { + "bodySha256": "730fdea9de97c82740161ed74f9ff189f756df5bf6390b411bc8ff25904dd6b5", + "bodyLen": 1024 + }, + { + "bodySha256": "5f454de71adfda348bff41dbfbab6b8c7672f81227e80654cb693597cdbae7b2", + "bodyLen": 1024 + }, + { + "bodySha256": "1f9ee3f2e96bd8524c480dc098c6ccea603c02e1e4cc0a9179681bbe9e76175d", + "bodyLen": 1024 + }, + { + "bodySha256": "6d2e417d8efe33aa08f8fe327e99dc9fe7ee3e917e0847161082125dc4fdaeb5", + "bodyLen": 1024 + }, + { + "bodySha256": "5b1b8b4834e13eba6f3bd1ca94b859339838caf7e4859d826e1d56d33129b76e", + "bodyLen": 1024 + }, + { + "bodySha256": "da9e996070cf61e9f8fe479bf7418b4ed8ab64a3df983affa6a55eeac3ea9580", + "bodyLen": 1024 + }, + { + "bodySha256": "2ed127277af74f542389da027697baa088f5f12e451c75b2149b6199347d8245", + "bodyLen": 1024 + }, + { + "bodySha256": "2d41b5d751481f7820d0743ff32090a38f55fe70c66524623d278ed8e9b7b28a", + "bodyLen": 1024 + }, + { + "bodySha256": "ed4e215fbb76d0214baeca11d1f7a3e65d03a3375db26e4587ec20a126a11c62", + "bodyLen": 1024 + }, + { + "bodySha256": "9d6647cca5ce62a465d4c0cdfb8b7377f160a5bd43cb0a3a6b5fd6f76b2de6d6", + "bodyLen": 1024 + }, + { + "bodySha256": "615549d8b8ce4b47d611f2d727e100db7951d2097822c48b5edb186fa7e60ec1", + "bodyLen": 1024 + }, + { + "bodySha256": "7bcbb78ccf799b50e25895da1e5fed0c1ab8c132f1152cbaeb41ed9448b1a822", + "bodyLen": 1024 + }, + { + "bodySha256": "fc815f3aae1762f39e157111b027c8a0bc4e98880ed05987d3aad06ce241f899", + "bodyLen": 1024 + }, + { + "bodySha256": "676c164914a90f7423b4d8ee4c866c59b4bc8b5927bfe47147bc7540376c0ea4", + "bodyLen": 1024 + }, + { + "bodySha256": "3983faa89ea71b7ed009adba4a0029e0d0e1d5dcbf42905d5653e042d992040e", + "bodyLen": 1024 + }, + { + "bodySha256": "6010e58c113525571f8e945ebf0589b8af9acaaf64f32ddc2ae969dce87cf429", + "bodyLen": 1024 + }, + { + "bodySha256": "10a9b4a58861ed8f25bda1982ac40b93d679be6b55ec82a24e100c336470de76", + "bodyLen": 1024 + }, + { + "bodySha256": "189c64125bb0a191e94c0d178727116aa84585c36c8d3515fb331bf5cff5e3f1", + "bodyLen": 1024 + }, + { + "bodySha256": "3f0b20b5940b01d1bc0ac7bbe72900286af8872ff39507b7b3be523cd4fe8108", + "bodyLen": 1024 + }, + { + "bodySha256": "b19cc5688c4615050070b7b340fc0728189bfa911bbbc69977a1c264555f9919", + "bodyLen": 1024 + }, + { + "bodySha256": "f2c59a7cc217decdd4439116ec80e118c6e0a10c9becb32d8ea350df8a7c5aa0", + "bodyLen": 1024 + }, + { + "bodySha256": "fa2d5e45dbb0deeb4083cbf3ec0e44c2fd14bf54e4289a776af5ac76dada8d53", + "bodyLen": 1024 + }, + { + "bodySha256": "acd943d4ca81033239ebb758115a54e24cdd542b35dcc0fc0df71ae5fa83a839", + "bodyLen": 1024 + }, + { + "bodySha256": "dcab78767216a10f1d5802ef99084e5b830d9e205ed614048f0b16ceda883e79", + "bodyLen": 1024 + }, + { + "bodySha256": "e8e88f5a24789271b3dfda94716759838c3397231f7942078288c6a3bc4bd704", + "bodyLen": 1024 + }, + { + "bodySha256": "11e02b9779b37dcaef326794c0af650d94e5ed2c8a5b857b7d6ef3e41bbd6a51", + "bodyLen": 1024 + }, + { + "bodySha256": "ea40b786c7d4eb239bb34c57a956850c89d6af15be3bf0d916d566d7a5eef3d8", + "bodyLen": 1024 + }, + { + "bodySha256": "1b9800f98fd63a1168cab9238ba582cb76241e27f2b1819a75dda65fe35b0b45", + "bodyLen": 1024 + }, + { + "bodySha256": "6ba36282806d776947072762cbbf41b6545649bbbc3134177efbadf72a5bf39c", + "bodyLen": 1024 + }, + { + "bodySha256": "782b0e199b97d19a49ee5b3394a1fdbfcc4fb72b5f687a171347b37742c9125a", + "bodyLen": 1024 + }, + { + "bodySha256": "e5d2d9923bfdf9ca813444b17eef777d77ad46cf96998ddd136320cbb8803a9e", + "bodyLen": 1024 + }, + { + "bodySha256": "8413305e981cba74dae45b53271b88e62f8783df59af25fc6b8b299b60b68d26", + "bodyLen": 1024 + }, + { + "bodySha256": "7cc238c46df22fdb69a143cc20a14b8baa62d8d21a2ec336baeb35e21b9ee623", + "bodyLen": 1024 + }, + { + "bodySha256": "f6d4e163e6f8ea5013e6d182ae580d6acf9f6a3a75c616bd4d7dcd4f52deda32", + "bodyLen": 1024 + }, + { + "bodySha256": "cf3d005ed1b114ab1989ac53eafb0487a87c34686976fb09b538cec5e53b43e2", + "bodyLen": 1024 + }, + { + "bodySha256": "2365e2bf1f908daed5ad914ba0948f229f775200a4151f24902b76305cc814fd", + "bodyLen": 1024 + }, + { + "bodySha256": "6b61da77caba2f62e669f863c12785b4b2eed9a6b16452db0bc131ceb71bd44e", + "bodyLen": 1024 + }, + { + "bodySha256": "f3b8eeab820cbade8cf198cfa5b74cdb00f4f008c0e26d0b20c6ddd5b1174415", + "bodyLen": 1024 + }, + { + "bodySha256": "d254fb87b13e7e3b4395314f982152722438f17d2668281f03b374a516be5b7f", + "bodyLen": 1024 + }, + { + "bodySha256": "2577f833a7aba0de1324bfec61e2ea118d768eff4ce961e98a3133712570f537", + "bodyLen": 1024 + }, + { + "bodySha256": "1eaf7f8c57ec30f64e398c75bb2bbac3efd2718dcbb6cc4c7f9bc1f850b3626f", + "bodyLen": 1024 + }, + { + "bodySha256": "7e07710523c884b203f05d7dd1da986512d6aa000eb86bef2661528dd1ff0cb2", + "bodyLen": 1024 + }, + { + "bodySha256": "6a836a4c8996e8b967dcccf11d02613231d20a58fb801317e642bcf153f1edd9", + "bodyLen": 1024 + }, + { + "bodySha256": "6d38e97b6a6cdb1c6d9e9bb3fbc4cada572a6ef479ed39e2c26f7a16ce4dcbee", + "bodyLen": 1024 + }, + { + "bodySha256": "29366226d88a91c78cb5756d69be06c052778e1d50b2893ddbe5e9824795524f", + "bodyLen": 1024 + }, + { + "bodySha256": "3dcfbcde750c11904da5a1284211ed4d3a9a7538c44a1d07c2a3efd92a7f2593", + "bodyLen": 1024 + }, + { + "bodySha256": "3e7185557615c599165b3b9206fc5fac1365ce992171431c5ccede170f518407", + "bodyLen": 1024 + }, + { + "bodySha256": "8ebe7134c9bd1995b836b2ea79ef27a393e786605d2a3eae603f0d6df35f4b7c", + "bodyLen": 1024 + }, + { + "bodySha256": "579ee1792454d428e683fc2348ec8452317d1bedda6d68e884a9ae015902be89", + "bodyLen": 1024 + }, + { + "bodySha256": "870dbc0b21eae1eb12398110ba374d9d9ac91f4a9607a806f832df899fa7bfc2", + "bodyLen": 1024 + }, + { + "bodySha256": "49cc32fa7ecc5fb4425917fdadeb5df6bceaceae81aab5ac6f289e7ef7891b9d", + "bodyLen": 1024 + }, + { + "bodySha256": "b2dc7d6cadf8b5f4a9ab0e32021fd274f260ed16e748021c7dd4f314341861a8", + "bodyLen": 1024 + }, + { + "bodySha256": "325b8cca06102096ef324c239e09b0833bca640825a72def4e98bc4213518f0d", + "bodyLen": 1024 + }, + { + "bodySha256": "b88407dabb5d6ad3feb57db352f6d0da0121f00ff82a5af969b683c5b49a77e3", + "bodyLen": 1024 + }, + { + "bodySha256": "ed53f94daf9d57e0b9873cd20085b71b6b71f4aad8c1d73a92db4a590f5e35d6", + "bodyLen": 1024 + }, + { + "bodySha256": "f0eb0be67fd4df6834c6c1601375ee12353fe4b021ac85dfdfcfefa12d928c05", + "bodyLen": 1024 + }, + { + "bodySha256": "8ddab31bfed345be5c1eb8c181f71fc597bf373310b432dbfecd4930d247073d", + "bodyLen": 1024 + }, + { + "bodySha256": "e5df2e2393de54e6bd79613a5fbc1c8617ebabe16fd829b229567659dfe23030", + "bodyLen": 1024 + }, + { + "bodySha256": "981a82ff267663756dfd30954f3cba84585fc9463f9f8fe611478212387bfe1d", + "bodyLen": 1024 + }, + { + "bodySha256": "2c6e35546f45d8cad3d1f27b38d9f3244f616f073489d35e8079f5ef8a4a0b28", + "bodyLen": 1024 + }, + { + "bodySha256": "4f516fc57dd2b96c431c588f628c18b88587a88384fd3a69671489d6fd946974", + "bodyLen": 1024 + }, + { + "bodySha256": "068535d25772074726a251fed7df91afd415caae4fea2634d0c2b814f89a5c84", + "bodyLen": 1024 + }, + { + "bodySha256": "18b32edd2f0de75650c32cbc7e08e597cfe317e24fb67ded5058cdc00fda8ddf", + "bodyLen": 1024 + }, + { + "bodySha256": "6149bb1a48fb734961b7090cb155eda01e0ffe87b7de8644f171f0953d8cb330", + "bodyLen": 1024 + }, + { + "bodySha256": "bb17f5d63e14e0133aa2905d09668f2d652dda717cdcb11d509c87298e7ce22a", + "bodyLen": 1024 + }, + { + "bodySha256": "4aeb3562345f0efe48cc79da37302a34233580b3d12fb7b022c9beeccc33bf84", + "bodyLen": 1024 + }, + { + "bodySha256": "1a23a8cb0f963510036f5059b40093a00b5383c6d63466ec75763f7733550d16", + "bodyLen": 1024 + }, + { + "bodySha256": "cf2892945eea8d25c1aa55e5ce26ce9f13412c4d212496bb0ebdacc2b63e7ef1", + "bodyLen": 1024 + }, + { + "bodySha256": "77e750816fb5cb802467341c1a58d9ae6cbcf82174a5402a4df52388c1dad2fc", + "bodyLen": 1024 + }, + { + "bodySha256": "285c12f4e8dbd234fd01ba2d033f8ca6b49b865997b4184804520c5744f0573c", + "bodyLen": 1024 + }, + { + "bodySha256": "584cedbfbdb3b7b77c007f13cf99cbf88088a7597c47496f07cc985d19dbf0fa", + "bodyLen": 1024 + }, + { + "bodySha256": "09fc9cb81e423575a9aa459d91112cfd414f925e89b90000fb8a76b4c774a3ce", + "bodyLen": 1024 + }, + { + "bodySha256": "d47e3588db1d7ee723ab2c87599cd9f5f1d7809ecfcf57de4d22c4427948d16f", + "bodyLen": 1024 + }, + { + "bodySha256": "f5c38acb6489c08bad5c14cf39a40b0549d30b96f6d3f7b21ded0a4ae7712c64", + "bodyLen": 1024 + }, + { + "bodySha256": "6befc2e02d84fb0dc0e2595b755c1cbd8633dbb85c5d10a44f2a55f5ea0bebce", + "bodyLen": 1024 + }, + { + "bodySha256": "9c372565e4efc808e2b89b70da44b1a6a92f53eb07d19f782c6d15a20e07c5d5", + "bodyLen": 1024 + }, + { + "bodySha256": "e303f44ecaf9c8bd60249fdb78ec3e8b5e8dbddc3b4cef7da84a93ad73d12ad5", + "bodyLen": 1024 + }, + { + "bodySha256": "6fafd12b22b3c720ee1440f2a84216abb75e3cba59600c69a95f21c15ad8c1b4", + "bodyLen": 1024 + }, + { + "bodySha256": "16c57ad3c95df216b39d3f7f0525def54f39ea4d28bb58623455255a131064a2", + "bodyLen": 1024 + }, + { + "bodySha256": "f60f9a3015c84ccf04e96256a424945d3316207f119d36079d5f6f8c9032b235", + "bodyLen": 1024 + }, + { + "bodySha256": "583d6f05ea7cbebb6a4b0c6326f0ca6cff573f43468a2f9ca1d05cda2f24ba75", + "bodyLen": 1024 + }, + { + "bodySha256": "b080dd75bf8fc5590a9e209e0df19eb194aaf67c0057e088e8c3beb7a7c21400", + "bodyLen": 1024 + }, + { + "bodySha256": "b289b069d05358e383d521f1b8f3d84ca4ea879bf3a8f29fd4e9d23b95c02586", + "bodyLen": 1024 + }, + { + "bodySha256": "5bfd65dbcbbaea67c5fc7b3e47cd5a4fe18fa876279c0b414994f5739834b358", + "bodyLen": 1024 + }, + { + "bodySha256": "e5e79b38d16296949789cd9b28029eaceafbd28d9927bc85ec5923a2a05e22b4", + "bodyLen": 1024 + }, + { + "bodySha256": "6b3d3bbb07e0622171963343d3cf8821e6325a180999366235e708d65a594d63", + "bodyLen": 1024 + }, + { + "bodySha256": "d9942076c73859a77c82d3acf0e2d47c03266373c16eeaab5e699f0af14959d8", + "bodyLen": 1024 + }, + { + "bodySha256": "bacb02c3aa407b0de2b69b37aab1fc4e0938de438bca1a306e9cc9e197f92ad3", + "bodyLen": 1024 + }, + { + "bodySha256": "12174ec348d0fc23001558d2bc67c68863930c9759d48389a09f253b38f65914", + "bodyLen": 1024 + }, + { + "bodySha256": "8ccfad1c14660035ab42b4246db92c53d9c4ff5be86e191c6ab1c15495e1a585", + "bodyLen": 1024 + }, + { + "bodySha256": "67cc998c951ec640889b651b3c9645a6b4c863ede9de9a542af0a9b24eb5fcb9", + "bodyLen": 1024 + }, + { + "bodySha256": "79764d7ae46ec09d4f3a62dc85c8da3ddc519a5bff4119244572610abdf98244", + "bodyLen": 1024 + }, + { + "bodySha256": "a11e066eb403788bdb23ff311687c24b1a594abd2714a8175dea59ff4f1f65e9", + "bodyLen": 1024 + }, + { + "bodySha256": "a55f8f8a94aad611e6e43fbba20374967a80ed6d3cb69cffd13008264d6ad034", + "bodyLen": 1024 + }, + { + "bodySha256": "e56a764916f7e21b389abea68d442c7fcd0ce89516ef90298d68a42081a9daa4", + "bodyLen": 1024 + }, + { + "bodySha256": "f57305fe396a61a7dc004375290639964f7befccbef840450488c6ec5bbe645f", + "bodyLen": 1024 + }, + { + "bodySha256": "f7be19d4682aec50ca0f07fa2b8cb777065097749e8bcf03b10ea8edd9480ff8", + "bodyLen": 1024 + }, + { + "bodySha256": "5d3130c12738449a51084dc84fe0b7fc528533fb696e43849ab3ff904dc59a28", + "bodyLen": 1024 + }, + { + "bodySha256": "5cbdc4622c1a62f7f81aadd9bfe6d57a095854564d950fa9c3c529b8f8e38533", + "bodyLen": 1024 + }, + { + "bodySha256": "1c16383c385771c608a6f2cbbc59a038da9b60afad8e92b3e0b2b87a27fce45c", + "bodyLen": 1024 + }, + { + "bodySha256": "70de3733991749aff1131ce8d6f6a53f74d07e9574c4abff2c510ee5c06af78f", + "bodyLen": 1024 + }, + { + "bodySha256": "3f532fe0b6ef911f8f5b0b98bbdd2afe4aeb108cca8363ebfc9099b611f92ecc", + "bodyLen": 1024 + }, + { + "bodySha256": "8703f59d467b9c17c3968135b2f077e1147eb3a60097d95615db3f23c8b8c8c3", + "bodyLen": 1024 + }, + { + "bodySha256": "b41cb97f48b43545eb7764727d8d5c750f32e1e69c79223a93982af44a7771b5", + "bodyLen": 1024 + }, + { + "bodySha256": "f3c50f4fd84ea109da2ade528879cbb5dc356c6f03de514b3c9dcbd719367238", + "bodyLen": 1024 + }, + { + "bodySha256": "ad10deaa1e503b1d514ae265e7b0818641997be414d687d29819148cfcbd16f6", + "bodyLen": 1024 + }, + { + "bodySha256": "9c3ea9a77bd34dde7828c5c84ff69238d893f8ce6b1d69d5c93fae4bf0870172", + "bodyLen": 1024 + }, + { + "bodySha256": "f6fe1ab7e2453b5f5324d5465ed9408f8ed1db886d911e9dd443b9f4fbb897fa", + "bodyLen": 1024 + }, + { + "bodySha256": "476447662532ceb4190254d0d70b8a4b6c6de18334f5bffb030b9f83369182a5", + "bodyLen": 1024 + }, + { + "bodySha256": "dad96a6cb845a10901175a95f72b35b6684e81cbfac6334b322d0b8a1adccd92", + "bodyLen": 1024 + }, + { + "bodySha256": "0af5cfde03eef345fd7c58494e767051ad763157c78d98cd9f95efa4f587598b", + "bodyLen": 1024 + }, + { + "bodySha256": "2b68cc6b4a03e35021c1c4253d798662bc3cf16973d7fad62790e6c0e7ea3285", + "bodyLen": 1024 + }, + { + "bodySha256": "f142b8f03d90b43b196c4b5c9dad544b1daa72e49019f12b2357bd5f0af15ff8", + "bodyLen": 1024 + }, + { + "bodySha256": "f32561cfb145b9947ebb582475370953525761a8089cd9feade7b874703c544c", + "bodyLen": 1024 + }, + { + "bodySha256": "b054643b6be7c58e4d98a91a8ef6ebad834545153b95231c356ad233c126b1c2", + "bodyLen": 1024 + }, + { + "bodySha256": "0f3e41d725c0398a1181f1a3dbb270f2877935651b4ea3088a2b19a09c519ae6", + "bodyLen": 1024 + }, + { + "bodySha256": "c00fc3845efbd826d2b692ba1443d0bdbdcd993a8db2c999518a15aa7769c24f", + "bodyLen": 1024 + }, + { + "bodySha256": "ee9a6ac07c844e1229e66cefce751691eee51f3b74ad2550e1b70322a7dd510f", + "bodyLen": 1024 + }, + { + "bodySha256": "ae0a348946f0620562c1c0a8a45a7629fd8179d172fb8d7bb619002d9c83b015", + "bodyLen": 1024 + }, + { + "bodySha256": "a4a93811465046891bd9f19e556c18fe0417cbde27c0d08b8a2db15d13e66753", + "bodyLen": 1024 + }, + { + "bodySha256": "76fa23a1d9dbefa535447b89c62616bff29579a6eddc10bb4c29601d36d27f5c", + "bodyLen": 1024 + }, + { + "bodySha256": "299fbfef79ed9cf43fd9233b15f79669ac017e2747821ee20208f39fb6a3d6fa", + "bodyLen": 1024 + }, + { + "bodySha256": "b6388ad4000158df9c76fc9dd6e77e3cf10c7f1a0719483efc200c2407412735", + "bodyLen": 1024 + }, + { + "bodySha256": "f00da5d1fda25d221c51490f08232b00c170683a60346d65c352edf54a84a84e", + "bodyLen": 1024 + }, + { + "bodySha256": "95877ab71cb0dd9971b9e2f06144bdbcdb6e7a90f8de04d8476f739f05bc2dfc", + "bodyLen": 1024 + }, + { + "bodySha256": "4eae70dd6b54903165f314f7162245b30aa0113b71fdba7d64b727557319aebf", + "bodyLen": 1024 + }, + { + "bodySha256": "e43432d571eb2afe14600cb94f0822a092f74e542570e2f82ca77f2162418075", + "bodyLen": 1024 + }, + { + "bodySha256": "02c2bfea0c3810549ab922e1d71d8cb96c0f320d91cab255028bc8c60cee51b3", + "bodyLen": 1024 + }, + { + "bodySha256": "0553bc01a03873f11a675e14722eafca7c601a07d03a9be93aefed487cfe5017", + "bodyLen": 1024 + }, + { + "bodySha256": "df471d4f0eb3789829a74736ffcfa4646fa0877dc71c4a0cb009734aff1db9e0", + "bodyLen": 1024 + }, + { + "bodySha256": "bf6cd3465cfdeda9056de5a3d234564ae04831df2a4c0f3cfe8ce2aa3f8c8454", + "bodyLen": 1024 + }, + { + "bodySha256": "d6fdf6ae1c5188f4f64dee8f2ef04226badc6dddf95fd275e8d42d07bb576b46", + "bodyLen": 1024 + }, + { + "bodySha256": "dbac846588d1de872bffdd7762d73b09af6f20d53fca60777357a03a58d50563", + "bodyLen": 1024 + }, + { + "bodySha256": "59d13b74879b2ef5c31e17303aa5265747217b32e9f45d485bdb364ab88b013e", + "bodyLen": 1024 + }, + { + "bodySha256": "1520851c20ac20a64cef204d5edd1b85cf9c56a8516f1c3de44be4e94257b28d", + "bodyLen": 1024 + }, + { + "bodySha256": "784c6c60553efc2ff249d0fbf4f4edb5699a89d4a5f17f3c83f46d4857c9da07", + "bodyLen": 1024 + }, + { + "bodySha256": "65ed09e32cae2f2048ce478bf1c9380cbb3926aed34994e9b94800dfe5f32684", + "bodyLen": 1024 + }, + { + "bodySha256": "84d9485bf2d25c68f9b679b3734592e04350b9755a85a99882e40334c3184fe7", + "bodyLen": 1024 + }, + { + "bodySha256": "5f8c063d02e482410919d0418bd45d1419402b045a5a69d3f85de5fb5cddae1f", + "bodyLen": 1024 + }, + { + "bodySha256": "10bbe3e6d226a0271483d2609831d352a765e32bc5a495486380a61625a7f583", + "bodyLen": 1024 + }, + { + "bodySha256": "4709dff67949ac5fbd2ff125ed7919e209db74cd15235e68f0915d0c1a908bda", + "bodyLen": 1024 + }, + { + "bodySha256": "8e7fd319121ffbb1436457e75f1766869d555d3e92631871f55555101a8defbb", + "bodyLen": 1024 + }, + { + "bodySha256": "742e87f080bcc8ef5de1ec631cbaed25bb05e1252d7248905f74f11b052486ae", + "bodyLen": 1024 + }, + { + "bodySha256": "ee1217cebef551ae2a5b27925f9d8d7bd9691a43631491ba937a8037dd62ca94", + "bodyLen": 1024 + }, + { + "bodySha256": "f04c9736dcc552d18b565098279a0428cbaec67479fe211f23290e3e83f3a5ba", + "bodyLen": 1024 + }, + { + "bodySha256": "4eb1a9f9401d09a9c252ca91330b77483ad63ac0b822cb723d16b1417cd72605", + "bodyLen": 1024 + }, + { + "bodySha256": "e17df085e4ebf6d37f7147f69ba22860dbb347c0201bdc87de7bd47a0b6c37e0", + "bodyLen": 1024 + }, + { + "bodySha256": "ca578403c90e264954f5ce0af4dc312b558e1193403c30b1dfeeb8104815a00b", + "bodyLen": 1024 + }, + { + "bodySha256": "8a8d77d74d67a58e41a6c115295cd943a494832c4104f90180b81d00e35223d2", + "bodyLen": 1024 + }, + { + "bodySha256": "9e355cf3658c395b1e6b669ba10ae2dec82975060600a01cadaf9ee35efb352e", + "bodyLen": 1024 + }, + { + "bodySha256": "bdcabe28e04486373ffc400e737304c65a3fe189dce857ec963bcfea541cb601", + "bodyLen": 1024 + }, + { + "bodySha256": "de5c1d7b2867253e737a612a20a3f319b5d15fe490424a85b40d3145f254112e", + "bodyLen": 1024 + }, + { + "bodySha256": "8260f387dd84cfb269cdf52d409fafb231a6d204d2396e88028b8f121b6ee7bb", + "bodyLen": 1024 + }, + { + "bodySha256": "0e75ed9a64adfdfd8ee926fcad48e918703b87482831f6cec9eb789e9ff746a8", + "bodyLen": 1024 + }, + { + "bodySha256": "32e7bb377f14f27b99c5ee4f5537d677193dffb673123e0e6dfea03044e2c65e", + "bodyLen": 1024 + }, + { + "bodySha256": "c96bd2b9e4256a097a9a4d5339a5772bd023b9afc14a3db787c6a07ceea9700a", + "bodyLen": 1024 + }, + { + "bodySha256": "402d90828397908a10f9e0e0bfd7d51e50b8f6f714fa9c1a9303789637d080ad", + "bodyLen": 1024 + }, + { + "bodySha256": "aadddb3e0cc6684f584f42198e7503704327683653ac55fd3395772f043d12d1", + "bodyLen": 1024 + }, + { + "bodySha256": "86fa59c05edfa757bd7278b8683ec5fa2311d47c2a8bd71590d3452ac25c8a9e", + "bodyLen": 1024 + }, + { + "bodySha256": "7d4d70329755c8a0d1cf728b5846c8d4fd1659ce87b62584e617119729fc809a", + "bodyLen": 1024 + }, + { + "bodySha256": "860899db820675a0d5b96f89ad259ab5c0099aa5315eda29c334295c6f6d90f7", + "bodyLen": 1024 + }, + { + "bodySha256": "0d158d4e1e1e107bd9ed27902be27ebc9712df7ae7db07de308ec3ee20ccfd49", + "bodyLen": 1024 + }, + { + "bodySha256": "bf3fbec630a864eda7f267d50ca017ccb9cff816dda75edc6e752e0838ed923b", + "bodyLen": 1024 + }, + { + "bodySha256": "2ce41919a206cef919b52c04a35c4df9c04adbdec5089c2871a5ccce10effaaa", + "bodyLen": 1024 + }, + { + "bodySha256": "74c65fd9f0907ffe44e6984dab4fda786de7fe43589323688b5c8876accd0458", + "bodyLen": 1024 + }, + { + "bodySha256": "1c873aee4f5e140b48b92b45c91f4d820c2086fbc08f03cb57094df711da70bc", + "bodyLen": 1024 + }, + { + "bodySha256": "20f5fbef50040e90390720483c2a5f4306739f4d4c2ff6aab049672935b3d5e8", + "bodyLen": 1024 + }, + { + "bodySha256": "14edef8f5382c3340e143b5f21787cc04a108c3182e45f06ae5825976d84e5a4", + "bodyLen": 1024 + }, + { + "bodySha256": "f09d5ce6dcf63d8aceb284257117fe2ef602b9ac29282360f3bd9660f6bb3851", + "bodyLen": 1024 + }, + { + "bodySha256": "b0136d38031825931793cf9d32f8691b1f92d8f8f616e2b49cb601934385bc26", + "bodyLen": 1024 + }, + { + "bodySha256": "471b25e014b151f3cecfd7362d0e8b25eae93b56ff742add3d66c1130753c7c3", + "bodyLen": 1024 + }, + { + "bodySha256": "4e3cab0825a621d6775d9a51795c858705b43e702c20becf9e9280d0134fc8d9", + "bodyLen": 1024 + }, + { + "bodySha256": "2a7676467e29c0691ce1f3db91d50c59bce4d4c0e4466c537f717cbb5932c0b7", + "bodyLen": 1024 + }, + { + "bodySha256": "0d7c0dc26c393add201ac8453bc87dd0e174f689a695ddbe9da2e8aead40df7e", + "bodyLen": 1024 + }, + { + "bodySha256": "b7f576aafaacfed3a8337cccde827d713a02830cfc733bf2813a1fbc9ce19737", + "bodyLen": 1024 + }, + { + "bodySha256": "f4f27213dc2495fd062b05e3cfe8306d9e29ffd840ac612f55b4cbc920c2866d", + "bodyLen": 1024 + }, + { + "bodySha256": "82c44dacc9370f9a279cda6cbb458e0aa63898e420866bc38bfe3f154283da86", + "bodyLen": 1024 + }, + { + "bodySha256": "48e353bcf02bdf7266f6741cdd7d74756f048f3abe7286bd2d727a3138455b20", + "bodyLen": 1024 + }, + { + "bodySha256": "776d761efae5ba8a533a0c83d3798d98c737745827f3e1997780df50b35b68b6", + "bodyLen": 1024 + }, + { + "bodySha256": "91e21e7909111339beaa246e9c1abab281cc31befd3492313b85f16d9b644035", + "bodyLen": 1024 + }, + { + "bodySha256": "37e3776f0b764882fe5e6eb1fee2e262cfd0eec8cdec8ac66e98d7f99827e195", + "bodyLen": 1024 + }, + { + "bodySha256": "7336023966e271f165846564d4ba9c9356ff4e0f2afaf1f7cf8f78736021da5b", + "bodyLen": 1024 + }, + { + "bodySha256": "7d040ccbbb431cdf0061898144f4125fcda0271872613b4c24050eaaf22368a1", + "bodyLen": 1024 + }, + { + "bodySha256": "6f32cc52cb6a5d83e5af31a26cd68704fca1cf5b5a6856d5aa119e16569ad1a7", + "bodyLen": 1024 + }, + { + "bodySha256": "4e2f7da2ac43a35524a667e3e773ddb92fd25efd160e4c0298e847425805f3a3", + "bodyLen": 1024 + }, + { + "bodySha256": "32af05cd088e0a2d6ca2467d0cb7cd6a5e2d2f1bb8401657b6cd3f989644a2df", + "bodyLen": 1024 + }, + { + "bodySha256": "c2a4129ee8c9772ce6154a4c3ecbc79319d1162bb0b9a7050feb89ba229a7177", + "bodyLen": 1024 + }, + { + "bodySha256": "c9baa66da7e1d1669858fbdaa1d1f02a635f8223a84418c430ecb3f26724bcd7", + "bodyLen": 1024 + }, + { + "bodySha256": "8ba4562f06b34e6e6b850cbe12553d0cc18eb5ded742f53ab4178d1c8bb6f011", + "bodyLen": 1024 + }, + { + "bodySha256": "6320daa4236de78bceb5b4fe5e09a6cc31dc1ac6a9fbf245ea21de8f3ccbaf8b", + "bodyLen": 1024 + }, + { + "bodySha256": "e5806986625dbade29a6cee23d93d0bea3dbbe290e8a19e8fed5d081875462be", + "bodyLen": 1024 + }, + { + "bodySha256": "575162ed31abc2163f1d3201a78c7ed8337dcdeb39d2cedeeb1fda951b02ca17", + "bodyLen": 1024 + }, + { + "bodySha256": "36b7af31dfd2f3f7805cf424d0d375ca2f5961f4c6cacbffe808cf26f5a3262b", + "bodyLen": 1024 + }, + { + "bodySha256": "be426c92c2d0a3c157fa8cb8826afc61ef10ddc586da6406701b443474405e2e", + "bodyLen": 1024 + }, + { + "bodySha256": "001827aa3d1d7702119578cbdc6680dfd16f7abe7b0826b73292c3edfa67ebf6", + "bodyLen": 1024 + }, + { + "bodySha256": "16d59c01f4b64c0d94e43bd00fb17b9c4ae2a3eb72bdfb4a5ae5959f6156639c", + "bodyLen": 1024 + }, + { + "bodySha256": "f190a9f65a10c44c05e1d980c5c3f737e6beafd963cd8bf368bf9e80168c56e8", + "bodyLen": 1024 + }, + { + "bodySha256": "657d57e9ef630d6b42927d0023096514064b29c2ece71762944440a472553542", + "bodyLen": 1024 + }, + { + "bodySha256": "4430d22a20c581706f88b6a1762c7580fe81fd0d2ca1cb667883ef64bb1984a4", + "bodyLen": 1024 + }, + { + "bodySha256": "81d21f94203daf23e72bc0ea2ffc543312b80b6d0afe0df10a781adb3690da2f", + "bodyLen": 1024 + }, + { + "bodySha256": "6971b915b2895018624e8d94a7be7a7242cb9d644fef408082c53fb832341194", + "bodyLen": 1024 + }, + { + "bodySha256": "9c326e18bbf7262be134e36aacf5f6226c94d5c94381f80367ce273a23dec5cb", + "bodyLen": 1024 + }, + { + "bodySha256": "e55a855934b3ed8b7e66bba82962eb195a01c5d4528470ca4ccd50c5bc2e617b", + "bodyLen": 1024 + }, + { + "bodySha256": "59b5ba75f47b41cacdb975a7c57d6609d69eae9045aaf26003b53a6c2f2ede61", + "bodyLen": 1024 + }, + { + "bodySha256": "56fa2cbc0b02e5fbd77eb8940cee089c1f4360affec44570480a020d09a00772", + "bodyLen": 1024 + }, + { + "bodySha256": "866806ee6bb52e0c5d70f40b7d8412c1634e58f7588c92381d819d2f6ae466f5", + "bodyLen": 1024 + }, + { + "bodySha256": "715d68f26bb8166288eee095836b95782042f6af1ba046e8b9d1a4a2d056c007", + "bodyLen": 1024 + }, + { + "bodySha256": "c6ba760db6942e4c060c082a4f88fb7a1ac0e59798c59fb7225e755e770aa9da", + "bodyLen": 1024 + }, + { + "bodySha256": "fbcd3f6697f0a94f38c493870ae1bfe3f2556b8fd878fc21915cf82c09f3b80f", + "bodyLen": 1024 + }, + { + "bodySha256": "5caac23664654c88de2dd7d8a3e0b4eb5dc89026fd34516258ff235e3bca08a7", + "bodyLen": 1024 + }, + { + "bodySha256": "0c3a259cd05e21ff88004e36171e47a4e0775b8a0a69f4a2a2405994b6930ae0", + "bodyLen": 1024 + }, + { + "bodySha256": "a3ef63aa355e289dcdca835cbc4ef06525b2fb1d1381a310dcfdbdea61ef97ab", + "bodyLen": 1024 + }, + { + "bodySha256": "2ec017ec8a729747f192c4238ca7b863c1b362ca4aeb5ca4a46f9eb516f558d3", + "bodyLen": 1024 + }, + { + "bodySha256": "a60533ca60d05d2302553efb1a3d0996e25eb4e2c54a8b17d4cf9baf2268be1f", + "bodyLen": 1024 + }, + { + "bodySha256": "805fbb6b141e1832530e87fb6ee12458d83f6eb01ab21b4065504e4531351af6", + "bodyLen": 1024 + }, + { + "bodySha256": "3790e9d5eabb94b41e0cd153122e66ebada9eec686c6c4a0c6414e2b4f9057d6", + "bodyLen": 1024 + }, + { + "bodySha256": "ac6c1cc3a395ad729daeea11a191f441438d26d9bc49acba514f1a37a32c90d5", + "bodyLen": 1024 + }, + { + "bodySha256": "87c1e8ad408765b0c45106dad3bb98efe17c6b66c96c2d958316aa086a4febd0", + "bodyLen": 1024 + }, + { + "bodySha256": "5353b1e0079021f39962c642bf4244ef64ebd9be7ffc9c46ce5f12310928bfe7", + "bodyLen": 1024 + }, + { + "bodySha256": "6f3fbda5e841e70e8efc84b3423159646c4c52abd00cce4dbe53101977cdbdc5", + "bodyLen": 1024 + }, + { + "bodySha256": "39c32158e79b88f6a8412086172eef1e0b3be8f92650d60456e974614aae3df6", + "bodyLen": 1024 + }, + { + "bodySha256": "a3b65aee33514a0b9a0c2513ebd71aa9ba6c667cb903a3bd0aa4049e58871eb4", + "bodyLen": 1024 + }, + { + "bodySha256": "f70df16fd7d493f8a7ac2fdcac22d7eb21c8d82e75a106b02be5ffd936cde60a", + "bodyLen": 1024 + }, + { + "bodySha256": "4ee1c9f63cd360409928bc4e51112b9dc1164ec3618411262c4d4547df641f63", + "bodyLen": 1024 + }, + { + "bodySha256": "c8e495ef7077b1e86319e41c40d519ae6ef2f4503b68c55f2b122bf969637dd6", + "bodyLen": 1024 + }, + { + "bodySha256": "9ed4d904c7025196afcba07dbb185d77b785a14ffb5a72f106cee36fdac4cc50", + "bodyLen": 1024 + }, + { + "bodySha256": "1a81fdb4cc5d8021a7e7e409da64f2692644000e130c36c8031acfb850d11896", + "bodyLen": 1024 + }, + { + "bodySha256": "4d3e6ba22aa552fe7215b0d810551074feef59c190c18c3947289f71a71aa6ba", + "bodyLen": 1024 + }, + { + "bodySha256": "08af32c41999ed9b7a4798562f3a7584ce53b7e043794f4dd0e41ae8b4860427", + "bodyLen": 1024 + }, + { + "bodySha256": "2c1b031fc1563a908bd17b1e483d1f6751df5a6a57ae5d3b3b6e45cfb8743495", + "bodyLen": 1024 + }, + { + "bodySha256": "8c7d17886be3c48a8cc949d8399347d8bbd7881840468d6162244f250ed863d8", + "bodyLen": 1024 + }, + { + "bodySha256": "1e5c2e9bc9e1549001df8a5d680d238a6c00c59fd57df75746ac1814f6ef908e", + "bodyLen": 1024 + }, + { + "bodySha256": "f875f7fe1f2b34943ec590364e43424690e72783e4c35325f6d92fa04f13aaab", + "bodyLen": 1024 + }, + { + "bodySha256": "edd77f6992c87e27f229b20ecb0e620389c7d416a19d469e444f13f366f4c268", + "bodyLen": 1024 + }, + { + "bodySha256": "7ef4d218565e7a7aa8e5fc6810e07f1805fe59b0919ad7ad06e86ce580abfb13", + "bodyLen": 1024 + }, + { + "bodySha256": "d005e2d941e60ee1eb5e8fea99f64f88002138311365675dd8694abed32d6a40", + "bodyLen": 1024 + }, + { + "bodySha256": "ca458f5fd752cbfd5142fdea580b964a32620867c965f66ae4c6ee29b7406e5c", + "bodyLen": 1024 + }, + { + "bodySha256": "12da75f2f6161d89e33e2b9beff7b6dedeb7dce5a65520cbada9c59e0ad79eb3", + "bodyLen": 1024 + }, + { + "bodySha256": "f753d0aed0f31fc3eb54c8e8517fc6da9abbf22cc208797a566e89f84c72da18", + "bodyLen": 1024 + }, + { + "bodySha256": "28153883cbc3b51835ea66902584ebb6ab7dd149af6a4d2614866ca106f58b9b", + "bodyLen": 1024 + }, + { + "bodySha256": "5e611aba81a078d4c7ed112303913104235e65624e9336f143a63362d24833d2", + "bodyLen": 1024 + }, + { + "bodySha256": "88f88f6af329d7cdd4d35498fd01453784c4c7a127eaa591870234df2dd884f2", + "bodyLen": 1024 + }, + { + "bodySha256": "7f5f6efc0b4bf9100bb957be0e91cbe1c45394fece3ba33b7746e7c1ad681053", + "bodyLen": 1024 + }, + { + "bodySha256": "5da1e1227ecdec0655291d38e4b673d511693b51f0c85bb13aa14a430ca912c4", + "bodyLen": 1024 + }, + { + "bodySha256": "a3d41ea55432298f9043981b46da467d6d4a28247d7b1c71a61478dbd0791503", + "bodyLen": 1024 + }, + { + "bodySha256": "91305f04fa3752d5e756730688e7ff0d2acfb8e005909633989f8e2923a68ac9", + "bodyLen": 1024 + }, + { + "bodySha256": "df8cfe7c48750324715b3c4c17b861c8c588d25969184b9335f4728dffba005f", + "bodyLen": 1024 + }, + { + "bodySha256": "e61f7690f5978f2a52b927eeaab5147ed83dfdbf686fdd1c984211df6d022a27", + "bodyLen": 1024 + } + ], + "salvage.plain": [ + { + "bodySha256": "4366db6349cc6064aa30857c8314b328ff7ab7d06d9662f75d463f60db00482e", + "bodyLen": 64 + }, + { + "bodySha256": "306a1849da49980d2d9531d3a5e278d560c347f178a4dd0a226bcd65809d384b", + "bodyLen": 64 + }, + { + "bodySha256": "c41e8336d47ff93fac7ec73db8ce1ce268f72d5bfa820f832c1e1c4643fe21ef", + "bodyLen": 64 + }, + { + "bodySha256": "9f60f224f52dac37b4efa340c732e70592929e5c263e62eb0661d80d1fb96ec4", + "bodyLen": 64 + }, + { + "bodySha256": "507c2ebbe0767fa78299aab305021a5af6992164a825b3ba13999fca957bd0c3", + "bodyLen": 64 + } + ], + "salvage.props": [ + { + "bodySha256": "d514897261fb458437274e486391d1d3b1fc3dac9bbbb36f1767b198d95af5fa", + "bodyLen": 64, + "props": { + "attempt": 1, + "region": "eu" + } + }, + { + "bodySha256": "30793dd7c8582707089584565e2c8b8da819e6f8557f64eb38fde62e1b397483", + "bodyLen": 64, + "props": { + "attempt": 1, + "region": "eu" + } + }, + { + "bodySha256": "4152762a250f173b69a3254a02b21c86902448b5f5fc0c44445c95515191826b", + "bodyLen": 64, + "props": { + "attempt": 1, + "region": "eu" + } + }, + { + "bodySha256": "9b4f25cfd8882d837289543d045e717c060dd99f3945ba7f4f9395731144f7b8", + "bodyLen": 64, + "props": { + "attempt": 1, + "region": "eu" + } + }, + { + "bodySha256": "7969a492d674202b941a66b40f86822660d7641a5fd1c72cd034a77d1c95f73a", + "bodyLen": 64, + "props": { + "attempt": 1, + "region": "eu" + } + } + ], + "salvage.scheduled": [ + { + "bodySha256": "db97f42afe16f7ad40c4c101cab07051a5e3754690258a72c40e886d223b6b0e", + "bodyLen": 64, + "scheduledAtMs": 4102444800000 + } + ] + } +} diff --git a/internal/journal/testdata_test.go b/internal/journal/testdata_test.go new file mode 100644 index 0000000..b458ef3 --- /dev/null +++ b/internal/journal/testdata_test.go @@ -0,0 +1,92 @@ +package journal + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +// fixtureDir extracts testdata/artemis-2.42-data.tar.gz into t.TempDir() and +// returns the extracted data/ path. It skips the test with a clear message if +// the tarball is missing (fixtures not yet harvested — run `make fixtures`). +func fixtureDir(t *testing.T) string { + t.Helper() + tarball := filepath.Join("testdata", "artemis-2.42-data.tar.gz") + f, err := os.Open(tarball) + if err != nil { + if os.IsNotExist(err) { + t.Skipf("fixture %s missing; run `make fixtures` to harvest it", tarball) + } + t.Fatalf("open fixture: %v", err) + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("gunzip fixture: %v", err) + } + defer gz.Close() + + dst := t.TempDir() + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("read fixture tar: %v", err) + } + name := filepath.Clean(hdr.Name) + if strings.HasPrefix(name, "..") || filepath.IsAbs(name) { + t.Fatalf("fixture tar has unsafe path %q", hdr.Name) + } + path := filepath.Join(dst, name) + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("extract dir %s: %v", name, err) + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("extract parent of %s: %v", name, err) + } + out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + t.Fatalf("extract file %s: %v", name, err) + } + if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // trusted committed fixture + out.Close() + t.Fatalf("extract file %s: %v", name, err) + } + if err := out.Close(); err != nil { + t.Fatalf("close extracted %s: %v", name, err) + } + default: + // Symlinks etc. are not expected in the fixture; fail loudly so a + // bad harvest is caught rather than silently skipped. + t.Fatalf("fixture tar has unexpected entry type %d for %q", hdr.Typeflag, hdr.Name) + } + } + return filepath.Join(dst, "data") +} + +// TestFixtureDir sanity-checks the committed fixture: it extracts and contains +// the four data subdirectories, and the journal is non-empty. +func TestFixtureDir(t *testing.T) { + dir := fixtureDir(t) + for _, sub := range []string{"bindings", "journal", "large-messages", "paging"} { + fi, err := os.Stat(filepath.Join(dir, sub)) + if err != nil || !fi.IsDir() { + t.Fatalf("fixture missing %s/: %v", sub, err) + } + } + entries, err := os.ReadDir(filepath.Join(dir, "journal")) + if err != nil || len(entries) == 0 { + t.Fatalf("fixture journal/ empty: %v", err) + } +} diff --git a/internal/journal/types.go b/internal/journal/types.go new file mode 100644 index 0000000..96e5887 --- /dev/null +++ b/internal/journal/types.go @@ -0,0 +1,267 @@ +package journal + +import ( + "fmt" + "math" + "unicode/utf16" +) + +// DataConstants type ids, per TypedProperties.encode / DataConstants (Artemis +// 2.42.0). See format_notes.md section 7. +const ( + dcNull = 0 + dcNotNull = 1 + dcBoolean = 2 + dcByte = 3 + dcBytes = 4 + dcShort = 5 + dcInt = 6 + dcLong = 7 + dcFloat = 8 + dcDouble = 9 + dcString = 10 + dcChar = 11 +) + +// reader is a bounds-checked big-endian cursor over a byte slice. All read +// methods are no-ops after the first error; check err() once at the end. +type reader struct { + b []byte + off int + e error +} + +// newReader wraps b in a reader positioned at offset 0. +func newReader(b []byte) *reader { + return &reader{b: b} +} + +// err returns the first error encountered by any read method, or nil. +func (r *reader) err() error { + return r.e +} + +// remaining returns the number of unread bytes. It is meaningless once err() +// is non-nil (the cursor stopped advancing at the failure point). +func (r *reader) remaining() int { + return len(r.b) - r.off +} + +// need reports whether n more bytes are available; on failure it sets a +// truncation error (if not already set) and returns false. +func (r *reader) need(n int) bool { + if r.e != nil { + return false + } + if n < 0 || r.off+n > len(r.b) { + r.e = fmt.Errorf("journal: truncated at offset %d", r.off) + return false + } + return true +} + +func (r *reader) u8() byte { + if !r.need(1) { + return 0 + } + v := r.b[r.off] + r.off++ + return v +} + +// bool decodes one byte; 0 = false, anything else (canonically 0xFF) = true. +func (r *reader) bool() bool { + return r.u8() != 0 +} + +func (r *reader) i16() int16 { + if !r.need(2) { + return 0 + } + v := int16(r.b[r.off])<<8 | int16(r.b[r.off+1]) + r.off += 2 + return v +} + +func (r *reader) i32() int32 { + if !r.need(4) { + return 0 + } + v := int32(r.b[r.off])<<24 | int32(r.b[r.off+1])<<16 | int32(r.b[r.off+2])<<8 | int32(r.b[r.off+3]) + r.off += 4 + return v +} + +func (r *reader) i64() int64 { + if !r.need(8) { + return 0 + } + var v int64 + for i := 0; i < 8; i++ { + v = v<<8 | int64(r.b[r.off+i]) + } + r.off += 8 + return v +} + +func (r *reader) f32() float32 { + bits := r.i32() + if r.e != nil { + return 0 + } + return math.Float32frombits(uint32(bits)) +} + +func (r *reader) f64() float64 { + bits := r.i64() + if r.e != nil { + return 0 + } + return math.Float64frombits(uint64(bits)) +} + +// bytes returns a sub-slice (not a copy) of the next n bytes. +func (r *reader) bytes(n int) []byte { + if !r.need(n) { + return nil + } + v := r.b[r.off : r.off+n] + r.off += n + return v +} + +// simpleString decodes a non-nullable SimpleString: a big-endian int +// byte-length followed by that many bytes of little-endian UTF-16 code-unit +// pairs (low byte first), per format_notes.md section 6. +func (r *reader) simpleString() string { + n := r.i32() + if r.e != nil { + return "" + } + if n < 0 { + r.e = fmt.Errorf("journal: negative SimpleString length at offset %d", r.off) + return "" + } + data := r.bytes(int(n)) + if r.e != nil { + return "" + } + if len(data)%2 != 0 { + r.e = fmt.Errorf("journal: odd SimpleString byte length at offset %d", r.off) + return "" + } + units := make([]uint16, len(data)/2) + for i := range units { + lo := data[2*i] + hi := data[2*i+1] + units[i] = uint16(lo) | uint16(hi)<<8 + } + return string(utf16.Decode(units)) +} + +// nullableSimpleString decodes a flag byte (0 = NULL, else NOT_NULL) followed +// by a SimpleString body when present. It returns ("", false) when NULL. +func (r *reader) nullableSimpleString() (string, bool) { + flag := r.u8() + if r.e != nil { + return "", false + } + if flag == 0 { + return "", false + } + s := r.simpleString() + if r.e != nil { + return "", false + } + return s, true +} + +// typedProperties decodes a TypedProperties block: a null-marker byte, then +// (if NOT_NULL) an int propertyCount and that many [key SimpleString][typed +// value] entries. It returns nil when the marker is NULL, per +// format_notes.md section 7. +func (r *reader) typedProperties() map[string]any { + marker := r.u8() + if r.e != nil { + return nil + } + if marker == dcNull { + return nil + } + + count := r.i32() + if r.e != nil { + return nil + } + if count < 0 { + r.e = fmt.Errorf("journal: negative TypedProperties count at offset %d", r.off) + return nil + } + + // Validate count won't cause OOM; minimum entry size is 5 bytes: + // 4-byte SimpleString length prefix + 1 type byte. + if count > int32(r.remaining()/5) { + r.e = fmt.Errorf("journal: TypedProperties count too large at offset %d", r.off) + return nil + } + + props := make(map[string]any, count) + for i := int32(0); i < count; i++ { + key := r.simpleString() + if r.e != nil { + return nil + } + typ := r.u8() + if r.e != nil { + return nil + } + + var val any + switch typ { + case dcNull: + val = nil + case dcBoolean: + val = r.bool() + case dcByte: + val = r.u8() + case dcBytes: + n := r.i32() + if r.e != nil { + return nil + } + if n < 0 { + r.e = fmt.Errorf("journal: negative BYTES length at offset %d", r.off) + return nil + } + raw := r.bytes(int(n)) + if r.e != nil { + return nil + } + cp := make([]byte, len(raw)) + copy(cp, raw) + val = cp + case dcShort: + val = r.i16() + case dcInt: + val = r.i32() + case dcLong: + val = r.i64() + case dcFloat: + val = r.f32() + case dcDouble: + val = r.f64() + case dcString: + val = r.simpleString() + case dcChar: + val = r.i16() + default: + r.e = fmt.Errorf("journal: unknown TypedProperties type id %d at offset %d", typ, r.off) + return nil + } + if r.e != nil { + return nil + } + props[key] = val + } + return props +} diff --git a/internal/journal/types_test.go b/internal/journal/types_test.go new file mode 100644 index 0000000..8d3296d --- /dev/null +++ b/internal/journal/types_test.go @@ -0,0 +1,477 @@ +package journal + +import ( + "bytes" + "encoding/binary" + "math" + "testing" +) + +// --- test helpers mirroring the verified on-disk layout (format_notes.md) --- + +// writeSimpleStringBytes appends a raw (non-nullable) SimpleString: a +// big-endian int byte-length followed by little-endian UTF-16 char pairs. +func writeSimpleStringBytes(buf *bytes.Buffer, s string) { + runes := []rune(s) + data := make([]byte, 0, len(runes)*2) + for _, r := range runes { + c := uint16(r) + data = append(data, byte(c&0xFF), byte(c>>8)) + } + var lenBuf [4]byte + binary.BigEndian.PutUint32(lenBuf[:], uint32(len(data))) + buf.Write(lenBuf[:]) + buf.Write(data) +} + +// writeNullableSimpleStringBytes appends a nullableSimpleString: flag byte +// (0 = NULL, 1 = NOT_NULL) followed by the SimpleString body when present. +func writeNullableSimpleStringBytes(buf *bytes.Buffer, s string, present bool) { + if !present { + buf.WriteByte(0) + return + } + buf.WriteByte(1) + writeSimpleStringBytes(buf, s) +} + +// writeTypedPropsHeader appends the TypedProperties null-marker + count. +func writeTypedPropsHeader(buf *bytes.Buffer, count int32) { + if count < 0 { + buf.WriteByte(0) // NULL marker -> empty/absent + return + } + buf.WriteByte(1) // NOT_NULL marker + var cntBuf [4]byte + binary.BigEndian.PutUint32(cntBuf[:], uint32(count)) + buf.Write(cntBuf[:]) +} + +// writeTypedPropKey appends a TypedProperties key: raw SimpleString, no flag. +func writeTypedPropKey(buf *bytes.Buffer, key string) { + writeSimpleStringBytes(buf, key) +} + +func be16(v int16) []byte { + var b [2]byte + binary.BigEndian.PutUint16(b[:], uint16(v)) + return b[:] +} + +func be32(v int32) []byte { + var b [4]byte + binary.BigEndian.PutUint32(b[:], uint32(v)) + return b[:] +} + +func be64(v int64) []byte { + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(v)) + return b[:] +} + +// --- SimpleString / nullableSimpleString round trip --- + +func TestReaderSimpleStringRoundTrip(t *testing.T) { + cases := []struct { + name string + s string + }{ + {"ascii", "hello"}, + {"empty", ""}, + {"nonascii", "héllo"}, + {"multibyte", "日本語"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + writeSimpleStringBytes(&buf, tc.s) + r := newReader(buf.Bytes()) + got := r.simpleString() + if err := r.err(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tc.s { + t.Fatalf("got %q, want %q", got, tc.s) + } + if r.remaining() != 0 { + t.Fatalf("remaining = %d, want 0", r.remaining()) + } + }) + } +} + +func TestReaderNullableSimpleStringPresent(t *testing.T) { + var buf bytes.Buffer + writeNullableSimpleStringBytes(&buf, "hello", true) + r := newReader(buf.Bytes()) + s, ok := r.nullableSimpleString() + if err := r.err(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ok { + t.Fatalf("expected present=true") + } + if s != "hello" { + t.Fatalf("got %q, want %q", s, "hello") + } +} + +func TestReaderNullableSimpleStringAbsent(t *testing.T) { + var buf bytes.Buffer + writeNullableSimpleStringBytes(&buf, "", false) + r := newReader(buf.Bytes()) + s, ok := r.nullableSimpleString() + if err := r.err(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ok { + t.Fatalf("expected present=false") + } + if s != "" { + t.Fatalf("got %q, want empty string", s) + } +} + +// --- TypedProperties --- + +func TestReaderTypedPropertiesEmpty(t *testing.T) { + var buf bytes.Buffer + writeTypedPropsHeader(&buf, 0) + r := newReader(buf.Bytes()) + props := r.typedProperties() + if err := r.err(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(props) != 0 { + t.Fatalf("got %d props, want 0", len(props)) + } +} + +func TestReaderTypedPropertiesNull(t *testing.T) { + var buf bytes.Buffer + writeTypedPropsHeader(&buf, -1) // NULL marker + r := newReader(buf.Bytes()) + props := r.typedProperties() + if err := r.err(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if props != nil { + t.Fatalf("got %#v, want nil", props) + } +} + +func TestReaderTypedPropertiesOneOfEach(t *testing.T) { + var buf bytes.Buffer + writeTypedPropsHeader(&buf, 11) + + // NULL + writeTypedPropKey(&buf, "kNull") + buf.WriteByte(0) + + // BOOLEAN true (encoded 0xFF per format_notes) + writeTypedPropKey(&buf, "kBool") + buf.WriteByte(2) + buf.WriteByte(0xFF) + + // BYTE + writeTypedPropKey(&buf, "kByte") + buf.WriteByte(3) + buf.WriteByte(0x7B) + + // BYTES + writeTypedPropKey(&buf, "kBytes") + buf.WriteByte(4) + payload := []byte{0x01, 0x02, 0x03} + buf.Write(be32(int32(len(payload)))) + buf.Write(payload) + + // SHORT + writeTypedPropKey(&buf, "kShort") + buf.WriteByte(5) + buf.Write(be16(-1234)) + + // INT + writeTypedPropKey(&buf, "kInt") + buf.WriteByte(6) + buf.Write(be32(-123456)) + + // LONG + writeTypedPropKey(&buf, "kLong") + buf.WriteByte(7) + buf.Write(be64(-123456789012)) + + // FLOAT + writeTypedPropKey(&buf, "kFloat") + buf.WriteByte(8) + buf.Write(be32(int32(math.Float32bits(3.14)))) + + // DOUBLE + writeTypedPropKey(&buf, "kDouble") + buf.WriteByte(9) + buf.Write(be64(int64(math.Float64bits(2.71828)))) + + // STRING + writeTypedPropKey(&buf, "kString") + buf.WriteByte(10) + writeSimpleStringBytes(&buf, "héllo") + + // CHAR + writeTypedPropKey(&buf, "kChar") + buf.WriteByte(11) + buf.Write(be16(int16('Z'))) + + r := newReader(buf.Bytes()) + props := r.typedProperties() + if err := r.err(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if r.remaining() != 0 { + t.Fatalf("remaining = %d, want 0", r.remaining()) + } + if len(props) != 11 { + t.Fatalf("got %d props, want 11: %#v", len(props), props) + } + + if v, ok := props["kNull"]; !ok || v != nil { + t.Errorf("kNull = %#v, want nil", v) + } + if v, ok := props["kBool"].(bool); !ok || v != true { + t.Errorf("kBool = %#v, want true", props["kBool"]) + } + if v, ok := props["kByte"].(byte); !ok || v != 0x7B { + t.Errorf("kByte = %#v, want 0x7B", props["kByte"]) + } + if v, ok := props["kBytes"].([]byte); !ok || !bytes.Equal(v, payload) { + t.Errorf("kBytes = %#v, want %#v", props["kBytes"], payload) + } + if v, ok := props["kShort"].(int16); !ok || v != -1234 { + t.Errorf("kShort = %#v, want -1234", props["kShort"]) + } + if v, ok := props["kInt"].(int32); !ok || v != -123456 { + t.Errorf("kInt = %#v, want -123456", props["kInt"]) + } + if v, ok := props["kLong"].(int64); !ok || v != -123456789012 { + t.Errorf("kLong = %#v, want -123456789012", props["kLong"]) + } + if v, ok := props["kFloat"].(float32); !ok || v != float32(3.14) { + t.Errorf("kFloat = %#v, want 3.14", props["kFloat"]) + } + if v, ok := props["kDouble"].(float64); !ok || v != 2.71828 { + t.Errorf("kDouble = %#v, want 2.71828", props["kDouble"]) + } + if v, ok := props["kString"].(string); !ok || v != "héllo" { + t.Errorf("kString = %#v, want héllo", props["kString"]) + } + if v, ok := props["kChar"].(int16); !ok || v != int16('Z') { + t.Errorf("kChar = %#v, want 'Z'", props["kChar"]) + } +} + +// --- primitive reader round trips --- + +func TestReaderPrimitives(t *testing.T) { + var buf bytes.Buffer + buf.WriteByte(0x42) // u8 + buf.WriteByte(0xFF) // bool true + buf.WriteByte(0x00) // bool false + buf.Write(be16(-100)) + buf.Write(be32(-100000)) + buf.Write(be64(-100000000000)) + buf.Write(be32(int32(math.Float32bits(1.5)))) + buf.Write(be64(int64(math.Float64bits(2.5)))) + buf.Write([]byte{0xAA, 0xBB, 0xCC}) + + r := newReader(buf.Bytes()) + if v := r.u8(); v != 0x42 { + t.Errorf("u8 = %#x, want 0x42", v) + } + if v := r.bool(); v != true { + t.Errorf("bool = %v, want true", v) + } + if v := r.bool(); v != false { + t.Errorf("bool = %v, want false", v) + } + if v := r.i16(); v != -100 { + t.Errorf("i16 = %d, want -100", v) + } + if v := r.i32(); v != -100000 { + t.Errorf("i32 = %d, want -100000", v) + } + if v := r.i64(); v != -100000000000 { + t.Errorf("i64 = %d, want -100000000000", v) + } + if v := r.f32(); v != 1.5 { + t.Errorf("f32 = %v, want 1.5", v) + } + if v := r.f64(); v != 2.5 { + t.Errorf("f64 = %v, want 2.5", v) + } + b := r.bytes(3) + if !bytes.Equal(b, []byte{0xAA, 0xBB, 0xCC}) { + t.Errorf("bytes = %#v, want AA BB CC", b) + } + if err := r.err(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if r.remaining() != 0 { + t.Fatalf("remaining = %d, want 0", r.remaining()) + } +} + +// --- truncation / bounds checking --- + +// TestReaderTruncatedSimpleString truncates a valid SimpleString buffer at +// every possible length and requires the reader to set an error without +// panicking, for every truncation point short of the full buffer. +func TestReaderTruncatedSimpleString(t *testing.T) { + var full bytes.Buffer + writeSimpleStringBytes(&full, "hello world") + data := full.Bytes() + + for n := 0; n < len(data); n++ { + t.Run("", func(t *testing.T) { + trunc := data[:n] + r := newReader(trunc) + func() { + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("panic at truncation %d: %v", n, rec) + } + }() + _ = r.simpleString() + }() + if r.err() == nil { + t.Fatalf("truncation at %d: expected error, got nil", n) + } + }) + } +} + +// TestReaderTruncatedNullableSimpleString does the same for the present case +// (flag byte = 1) of nullableSimpleString. +func TestReaderTruncatedNullableSimpleString(t *testing.T) { + var full bytes.Buffer + writeNullableSimpleStringBytes(&full, "hello world", true) + data := full.Bytes() + + for n := 0; n < len(data); n++ { + t.Run("", func(t *testing.T) { + trunc := data[:n] + r := newReader(trunc) + func() { + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("panic at truncation %d: %v", n, rec) + } + }() + _, _ = r.nullableSimpleString() + }() + if r.err() == nil { + t.Fatalf("truncation at %d: expected error, got nil", n) + } + }) + } +} + +// TestReaderTruncatedTypedProperties builds a one-of-each-type TypedProperties +// buffer and requires every truncation length 0..len-1 to produce an error, +// never a panic. +func TestReaderTruncatedTypedProperties(t *testing.T) { + var full bytes.Buffer + writeTypedPropsHeader(&full, 3) + + writeTypedPropKey(&full, "kBool") + full.WriteByte(2) + full.WriteByte(0xFF) + + writeTypedPropKey(&full, "kBytes") + full.WriteByte(4) + payload := []byte{0x01, 0x02, 0x03, 0x04} + full.Write(be32(int32(len(payload)))) + full.Write(payload) + + writeTypedPropKey(&full, "kString") + full.WriteByte(10) + writeSimpleStringBytes(&full, "world") + + data := full.Bytes() + + for n := 0; n < len(data); n++ { + t.Run("", func(t *testing.T) { + trunc := data[:n] + r := newReader(trunc) + func() { + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("panic at truncation %d: %v", n, rec) + } + }() + _ = r.typedProperties() + }() + if r.err() == nil { + t.Fatalf("truncation at %d: expected error, got nil", n) + } + }) + } +} + +// TestReaderTypedPropertiesOOMRegression verifies that a malformed count +// (corrupted journal input: NOT_NULL marker + count = 0x7FFFFFFF) does not +// trigger fatal OOM allocation before buffer validation, but instead sets +// an error and returns nil. +func TestReaderTypedPropertiesOOMRegression(t *testing.T) { + // Minimal corrupted input: NOT_NULL marker (1 byte) + count 0x7FFFFFFF (4 bytes). + // This is only 5 bytes, far too small for 2147483647 entries. + malformed := []byte{ + 0x01, // NOT_NULL marker + 0x7F, 0xFF, 0xFF, 0xFF, // 0x7FFFFFFF in big-endian + } + r := newReader(malformed) + func() { + defer func() { + if rec := recover(); rec != nil { + t.Fatalf("OOM crash (or panic) with malformed count: %v", rec) + } + }() + props := r.typedProperties() + if props != nil { + t.Fatalf("expected nil, got %#v", props) + } + }() + if r.err() == nil { + t.Fatalf("expected error, got nil") + } +} + +// TestReaderErrIsSticky verifies that once an error occurs, subsequent reads +// are no-ops (return zero values) rather than panicking or advancing. +func TestReaderErrIsSticky(t *testing.T) { + r := newReader([]byte{0x01}) + _ = r.i32() // underflow: only 1 byte available + if r.err() == nil { + t.Fatalf("expected error after underflow read") + } + // Further reads must not panic and must return zero values. + if v := r.u8(); v != 0 { + t.Errorf("u8 after error = %#x, want 0", v) + } + if v := r.i64(); v != 0 { + t.Errorf("i64 after error = %d, want 0", v) + } + if v := r.simpleString(); v != "" { + t.Errorf("simpleString after error = %q, want empty", v) + } + if b := r.bytes(5); b != nil { + t.Errorf("bytes after error = %#v, want nil", b) + } +} + +func TestReaderBytesIsSubSlice(t *testing.T) { + data := []byte{1, 2, 3, 4, 5} + r := newReader(data) + b := r.bytes(3) + if &b[0] != &data[0] { + t.Fatalf("bytes() did not return a sub-slice of the original backing array") + } +} diff --git a/internal/store/checkpoint.go b/internal/store/checkpoint.go new file mode 100644 index 0000000..0f5ddd9 --- /dev/null +++ b/internal/store/checkpoint.go @@ -0,0 +1,34 @@ +package store + +import ( + "os" + "strconv" + "strings" +) + +// ckptPath is the sidecar checkpoint file for a store: ".ckpt". +func ckptPath(storePath string) string { return storePath + ".ckpt" } + +// LoadCheckpoint returns the last saved redelivery offset for storePath, or 0 +// if no checkpoint exists yet (a fresh replay starts from the first record). +func LoadCheckpoint(storePath string) (int64, error) { + b, err := os.ReadFile(ckptPath(storePath)) + if os.IsNotExist(err) { + return 0, nil + } + if err != nil { + return 0, err + } + return strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64) +} + +// SaveCheckpoint durably records offset as the redelivery progress for +// storePath. It writes to a temp file and renames it into place so a crash +// mid-write cannot corrupt the checkpoint. +func SaveCheckpoint(storePath string, offset int64) error { + tmp := ckptPath(storePath) + ".tmp" + if err := os.WriteFile(tmp, []byte(strconv.FormatInt(offset, 10)), 0o600); err != nil { + return err + } + return os.Rename(tmp, ckptPath(storePath)) +} diff --git a/internal/store/checkpoint_test.go b/internal/store/checkpoint_test.go new file mode 100644 index 0000000..883f3b6 --- /dev/null +++ b/internal/store/checkpoint_test.go @@ -0,0 +1,21 @@ +// internal/store/checkpoint_test.go +package store + +import ( + "path/filepath" + "testing" +) + +func TestCheckpointRoundTrip(t *testing.T) { + sp := filepath.Join(t.TempDir(), "s.artx") + if off, err := LoadCheckpoint(sp); err != nil || off != 0 { + t.Fatalf("empty checkpoint want 0, got %d err %v", off, err) + } + if err := SaveCheckpoint(sp, 12345); err != nil { + t.Fatal(err) + } + off, err := LoadCheckpoint(sp) + if err != nil || off != 12345 { + t.Fatalf("want 12345 got %d err %v", off, err) + } +} diff --git a/internal/store/core_record_test.go b/internal/store/core_record_test.go new file mode 100644 index 0000000..297ca61 --- /dev/null +++ b/internal/store/core_record_test.go @@ -0,0 +1,107 @@ +package store + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "io" + "os" + "path/filepath" + "testing" +) + +func TestStoreCoreAndAMQPRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "mix.artx") + w, err := NewWriter(path) + if err != nil { + t.Fatalf("new writer: %v", err) + } + amqpRec := Record{UUID: [16]byte{1}, Queue: "a", DrainedAt: 10, Kind: KindAMQP, AMQP: []byte("amqp-bytes")} + coreRec := Record{UUID: [16]byte{2}, Queue: "b", DrainedAt: 20, Kind: KindCore, CorePayload: []byte("core-payload")} + for _, r := range []Record{amqpRec, coreRec} { + if err := w.Append(r); err != nil { + t.Fatalf("append: %v", err) + } + } + if err := w.Sync(); err != nil { + t.Fatalf("sync: %v", err) + } + _ = w.Close() + + r, err := OpenReader(path) + if err != nil { + t.Fatalf("open: %v", err) + } + defer r.Close() + + got := readAll(t, r) + if len(got) != 2 { + t.Fatalf("want 2 records, got %d", len(got)) + } + if got[0].Kind != KindAMQP || string(got[0].AMQP) != "amqp-bytes" || len(got[0].CorePayload) != 0 { + t.Errorf("AMQP record round-trip mismatch: %+v", got[0]) + } + if got[1].Kind != KindCore || string(got[1].CorePayload) != "core-payload" || len(got[1].AMQP) != 0 { + t.Errorf("Core record round-trip mismatch: %+v", got[1]) + } +} + +// TestStoreReadsVersion1 hand-writes a legacy v1 store (no per-record Kind +// byte) and confirms the reader still decodes it, defaulting to KindAMQP. +func TestStoreReadsVersion1(t *testing.T) { + path := filepath.Join(t.TempDir(), "v1.artx") + + var buf bytes.Buffer + buf.WriteString(Magic) + buf.WriteByte(Version1) + + // v1 body: UUID(16) + DrainedAt(8) + qlen(2)+queue + alen(4)+amqp + uuid := [16]byte{9} + queue := "legacy" + amqp := []byte("v1-amqp") + body := append([]byte(nil), uuid[:]...) + body = binary.BigEndian.AppendUint64(body, 7) + body = binary.BigEndian.AppendUint16(body, uint16(len(queue))) + body = append(body, queue...) + body = binary.BigEndian.AppendUint32(body, uint32(len(amqp))) + body = append(body, amqp...) + + var hdr [4]byte + binary.BigEndian.PutUint32(hdr[:], uint32(len(body))) + buf.Write(hdr[:]) + buf.Write(body) + _ = binary.Write(&buf, binary.BigEndian, crc32.ChecksumIEEE(body)) + + if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil { + t.Fatalf("write v1 store: %v", err) + } + + r, err := OpenReader(path) + if err != nil { + t.Fatalf("open v1: %v", err) + } + defer r.Close() + got := readAll(t, r) + if len(got) != 1 { + t.Fatalf("want 1 record, got %d", len(got)) + } + if got[0].Kind != KindAMQP || got[0].Queue != "legacy" || string(got[0].AMQP) != "v1-amqp" { + t.Errorf("v1 record mismatch: %+v", got[0]) + } +} + +func readAll(t *testing.T, r *Reader) []Record { + t.Helper() + var out []Record + for { + rec, _, err := r.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("next: %v", err) + } + out = append(out, rec) + } + return out +} diff --git a/internal/store/dedup.go b/internal/store/dedup.go new file mode 100644 index 0000000..d662565 --- /dev/null +++ b/internal/store/dedup.go @@ -0,0 +1,67 @@ +package store + +import ( + "crypto/sha256" + + "github.com/Azure/go-amqp" +) + +// DedupID returns a stable 16-byte content id for a message on a given +// queue, excluding the transport fields Artemis mutates on redelivery — +// Header.DeliveryCount and Header.FirstAcquirer, plus the +// delivery-annotations section — so a re-drained redelivered message hashes +// identically to its first drain. Everything that is part of the message's +// identity (durable/priority/ttl, message annotations, properties, +// application-properties, body, footer) is kept, so two messages that differ +// only in, say, priority still get different ids. +// +// It hashes a shallow clone with a fresh Header copy (the volatile fields +// zeroed) and the delivery annotations cleared, so the original msg is never +// mutated and stays full-fidelity for replay. +// +// The normalized bytes are then salted with queue: salvage fans one journal +// message out to N queues, and an unsalted content hash would collide across +// those queues, causing the broker to drop N-1 copies as duplicates on +// redeliver. Salting with the queue name gives each fanned-out copy its own +// id while a same-queue crash re-drain (queue identical) still collides onto +// one id, preserving the invariant the WAL depends on. +func DedupID(msg *amqp.Message, queue string) [16]byte { + clone := *msg + if msg.Header != nil { + h := *msg.Header + h.DeliveryCount = 0 + h.FirstAcquirer = false + clone.Header = &h + } + clone.DeliveryAnnotations = nil + var id [16]byte + raw, err := clone.MarshalBinary() + if err != nil { + // A message that already marshaled successfully for rec.AMQP cannot + // fail to marshal here after only clearing fields; fall back to hashing + // the original bytes so we still produce a deterministic id. + raw, _ = msg.MarshalBinary() + } + sum := sha256.New() + sum.Write(raw) // normalized marshaled message + sum.Write([]byte{0}) // domain separator + sum.Write([]byte(queue)) + digest := sum.Sum(nil) + copy(id[:], digest[:16]) + return id +} + +// DedupIDCore returns a stable 16-byte content id for a Core message (which +// has no amqp.Message form) from its serialized Core payload salted with the +// queue name, mirroring DedupID's domain separation so a same-queue re-drain +// collides onto one id while a fan-out to N queues gets N distinct ids. +func DedupIDCore(payload []byte, queue string) [16]byte { + var id [16]byte + sum := sha256.New() + sum.Write(payload) + sum.Write([]byte{1}) // domain separator (distinct from DedupID's 0) + sum.Write([]byte(queue)) + digest := sum.Sum(nil) + copy(id[:], digest[:16]) + return id +} diff --git a/internal/store/dedup_test.go b/internal/store/dedup_test.go new file mode 100644 index 0000000..c9be539 --- /dev/null +++ b/internal/store/dedup_test.go @@ -0,0 +1,81 @@ +package store + +import ( + "testing" + + "github.com/Azure/go-amqp" +) + +func TestDedupID(t *testing.T) { + newMsg := func(body string) *amqp.Message { + return amqp.NewMessage([]byte(body)) + } + + t.Run("same message and queue produce equal ids", func(t *testing.T) { + m1 := newMsg("hello") + m2 := newMsg("hello") + if DedupID(m1, "orders") != DedupID(m2, "orders") { + t.Fatalf("expected equal ids for identical message+queue") + } + }) + + t.Run("redelivery bump does not change the id (crash re-drain invariant)", func(t *testing.T) { + m1 := newMsg("hello") + m1.Header = &amqp.MessageHeader{Durable: true} + + m2 := newMsg("hello") + m2.Header = &amqp.MessageHeader{Durable: true, DeliveryCount: 1, FirstAcquirer: true} + m2.DeliveryAnnotations = amqp.Annotations{"x-opt-delivery-count": 1} + + if DedupID(m1, "orders") != DedupID(m2, "orders") { + t.Fatalf("expected equal ids across a redelivery bump (DeliveryCount/FirstAcquirer/DeliveryAnnotations)") + } + }) + + t.Run("same message, different queue produce different ids", func(t *testing.T) { + m1 := newMsg("hello") + m2 := newMsg("hello") + if DedupID(m1, "orders") == DedupID(m2, "shipping") { + t.Fatalf("expected different ids for the same message fanned out to different queues") + } + }) + + t.Run("different body, same queue produce different ids", func(t *testing.T) { + m1 := newMsg("hello") + m2 := newMsg("goodbye") + if DedupID(m1, "orders") == DedupID(m2, "orders") { + t.Fatalf("expected different ids for different message bodies") + } + }) +} + +func TestDedupIDCore(t *testing.T) { + payload := []byte("core-payload-bytes") + + t.Run("deterministic for same payload+queue", func(t *testing.T) { + if DedupIDCore(payload, "orders") != DedupIDCore(payload, "orders") { + t.Fatalf("expected equal ids for identical payload+queue") + } + }) + + t.Run("same payload, different queue produce different ids", func(t *testing.T) { + if DedupIDCore(payload, "orders") == DedupIDCore(payload, "shipping") { + t.Fatalf("expected different ids for the same payload fanned out to different queues") + } + }) + + t.Run("different payload, same queue produce different ids", func(t *testing.T) { + if DedupIDCore(payload, "orders") == DedupIDCore([]byte("other"), "orders") { + t.Fatalf("expected different ids for different payloads") + } + }) + + t.Run("core domain separated from amqp id", func(t *testing.T) { + // The Core domain byte (1) must keep a Core payload from ever colliding + // with an AMQP id (domain byte 0) over the same raw bytes and queue. + m := amqp.NewMessage(payload) + if DedupIDCore(payload, "orders") == DedupID(m, "orders") { + t.Fatalf("Core id collided with AMQP id despite domain separation") + } + }) +} diff --git a/internal/store/format.go b/internal/store/format.go new file mode 100644 index 0000000..cdaa38d --- /dev/null +++ b/internal/store/format.go @@ -0,0 +1,42 @@ +// Package store is the on-disk format for drained messages: an append-only +// write-ahead log (WAL) chosen so an evacuation that crashes mid-write is still +// readable up to the last intact record. Writer appends and fsyncs records +// (writer.go), Reader streams them back for redelivery (reader.go), and a +// sidecar checkpoint tracks replay progress so an interrupted redelivery +// resumes (checkpoint.go). This file defines the shared constants and record +// shape; the exact byte layout lives alongside Writer/Reader. +package store + +const ( + Magic = "ARTX" // 4-byte file signature at offset 0 + Version byte = 2 // format version, one byte after the magic + + // Version1 stores AMQP records only, with no per-record Kind byte. Reader + // still accepts it (every v1 record is implicitly KindAMQP). + Version1 byte = 1 + headerLen = 5 // len(Magic) + 1 version byte +) + +// Record kinds (v2+). KindAMQP records carry raw AMQP wire bytes in AMQP; +// KindCore records carry a serialized Core payload in CorePayload (which +// redeliver converts to AMQP on send — Core cannot be sent over the wire +// directly by this AMQP-only client). +const ( + KindAMQP byte = 0 + KindCore byte = 1 +) + +// Record is one drained message as held in the WAL. UUID is the deterministic +// content id reused as _AMQ_DUPL_ID on redelivery to defeat duplicates; Queue +// is the originating queue; DrainedAt is the drain time in unix nanoseconds. +// Kind selects the payload: KindAMQP uses AMQP (raw amqp.Message wire encoding, +// replayed verbatim); KindCore uses CorePayload (a serialized journal +// CorePayload, converted to AMQP at redeliver time). +type Record struct { + UUID [16]byte + Queue string + DrainedAt int64 + Kind byte + AMQP []byte + CorePayload []byte +} diff --git a/internal/store/reader.go b/internal/store/reader.go new file mode 100644 index 0000000..b8e45e9 --- /dev/null +++ b/internal/store/reader.go @@ -0,0 +1,142 @@ +package store + +import ( + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io" + "os" +) + +// ErrCorrupt wraps any record that fails to decode — a bad length prefix, a +// CRC mismatch, or a truncated tail. A caller that hits it has read every +// intact record up to that offset. +var ErrCorrupt = errors.New("store: corrupt or truncated record") + +// Reader streams records out of a WAL store file in order. It validates the +// header on open and each record's CRC on read; call Next until io.EOF, or +// SeekTo an offset first to resume. A Reader is not safe for concurrent use. +type Reader struct { + f *os.File + offset int64 + size int64 + version byte +} + +// OpenReader opens path and verifies the store magic and version, returning a +// Reader positioned just after the header. +func OpenReader(path string) (*Reader, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open store: %w", err) + } + hdr := make([]byte, headerLen) + if _, err := io.ReadFull(f, hdr); err != nil { + _ = f.Close() + return nil, fmt.Errorf("read header: %w", err) + } + if string(hdr[:4]) != Magic || (hdr[4] != Version && hdr[4] != Version1) { + _ = f.Close() + return nil, fmt.Errorf("bad store header (magic %q version %d)", hdr[:4], hdr[4]) + } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, fmt.Errorf("stat store: %w", err) + } + return &Reader{f: f, offset: headerLen, size: fi.Size(), version: hdr[4]}, nil +} + +// SeekTo positions the reader at offset so the next Next reads the record +// there; it is used to resume a redelivery from a saved checkpoint. An offset +// below the header is clamped to the first record. +func (r *Reader) SeekTo(offset int64) error { + if offset < headerLen { + offset = headerLen + } + if _, err := r.f.Seek(offset, io.SeekStart); err != nil { + return err + } + r.offset = offset + return nil +} + +// Next reads and returns the record at the current position, along with the +// offset just past it (the value to checkpoint after redelivering it). It +// returns io.EOF at a clean end of file, or an error wrapping ErrCorrupt if the +// record is truncated or its CRC fails. +func (r *Reader) Next() (Record, int64, error) { + var lenBuf [4]byte + n, err := io.ReadFull(r.f, lenBuf[:]) + if n == 0 && errors.Is(err, io.EOF) { + return Record{}, r.offset, io.EOF + } + if err != nil { + return Record{}, r.offset, fmt.Errorf("%w: length prefix: %w", ErrCorrupt, err) + } + bodyLen := binary.BigEndian.Uint32(lenBuf[:]) + maxBody := r.size - (r.offset + 4) - 4 + if maxBody < 0 || int64(bodyLen) > maxBody { + return Record{}, r.offset, fmt.Errorf("%w: record length %d exceeds remaining file bytes", ErrCorrupt, bodyLen) + } + body := make([]byte, bodyLen) + if _, err := io.ReadFull(r.f, body); err != nil { + return Record{}, r.offset, fmt.Errorf("%w: body: %w", ErrCorrupt, err) + } + var crcBuf [4]byte + if _, err := io.ReadFull(r.f, crcBuf[:]); err != nil { + return Record{}, r.offset, fmt.Errorf("%w: crc: %w", ErrCorrupt, err) + } + if binary.BigEndian.Uint32(crcBuf[:]) != crc32.ChecksumIEEE(body) { + return Record{}, r.offset, fmt.Errorf("%w: crc mismatch at offset %d", ErrCorrupt, r.offset) + } + + rec, err := decodeBody(body, r.version) + if err != nil { + return Record{}, r.offset, fmt.Errorf("%w: %w", ErrCorrupt, err) + } + r.offset += int64(4 + bodyLen + 4) + return rec, r.offset, nil +} + +func decodeBody(body []byte, version byte) (Record, error) { + var rec Record + // v2+ leads with a 1-byte Kind; v1 has no Kind byte (implicitly KindAMQP). + if version >= Version { + if len(body) < 1 { + return Record{}, errors.New("body too short") + } + rec.Kind = body[0] + body = body[1:] + } + if len(body) < 16+8+2 { + return Record{}, errors.New("body too short") + } + copy(rec.UUID[:], body[:16]) + p := 16 + rec.DrainedAt = int64(binary.BigEndian.Uint64(body[p : p+8])) + p += 8 + qlen := int(binary.BigEndian.Uint16(body[p : p+2])) + p += 2 + if p+qlen+4 > len(body) { + return Record{}, errors.New("queue length overflow") + } + rec.Queue = string(body[p : p+qlen]) + p += qlen + alen := int(binary.BigEndian.Uint32(body[p : p+4])) + p += 4 + if p+alen != len(body) { + return Record{}, errors.New("payload length overflow") + } + payload := append([]byte(nil), body[p:p+alen]...) + if rec.Kind == KindCore { + rec.CorePayload = payload + } else { + rec.AMQP = payload + } + return rec, nil +} + +// Close closes the underlying store file. +func (r *Reader) Close() error { return r.f.Close() } diff --git a/internal/store/reader_test.go b/internal/store/reader_test.go new file mode 100644 index 0000000..28bad53 --- /dev/null +++ b/internal/store/reader_test.go @@ -0,0 +1,116 @@ +// internal/store/reader_test.go +package store + +import ( + "encoding/binary" + "errors" + "io" + "os" + "path/filepath" + "testing" + "time" +) + +func writeRecords(t *testing.T, path string, recs []Record) { + t.Helper() + w, err := NewWriter(path) + if err != nil { + t.Fatal(err) + } + for _, r := range recs { + if err := w.Append(r); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } +} + +func TestReaderRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "s.artx") + in := []Record{ + {UUID: [16]byte{1}, Queue: "orders", DrainedAt: 10, AMQP: []byte("a")}, + {UUID: [16]byte{2}, Queue: "payments", DrainedAt: 20, AMQP: []byte("bb")}, + } + writeRecords(t, path, in) + + r, err := OpenReader(path) + if err != nil { + t.Fatal(err) + } + defer r.Close() + var got []Record + for { + rec, _, err := r.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("next: %v", err) + } + got = append(got, rec) + } + if len(got) != 2 || got[0].Queue != "orders" || string(got[1].AMQP) != "bb" { + t.Fatalf("round-trip mismatch: %+v", got) + } +} + +func TestReaderDetectsTruncation(t *testing.T) { + path := filepath.Join(t.TempDir(), "s.artx") + writeRecords(t, path, []Record{{UUID: [16]byte{1}, Queue: "orders", DrainedAt: 10, AMQP: []byte("hello")}}) + // Chop the last 3 bytes to simulate a crash mid-write. + b, _ := os.ReadFile(path) + _ = os.WriteFile(path, b[:len(b)-3], 0o600) + + r, err := OpenReader(path) + if err != nil { + t.Fatal(err) + } + defer r.Close() + _, _, err = r.Next() + if !errors.Is(err, ErrCorrupt) && !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("expected corruption/truncation error, got %v", err) + } +} + +// TestReaderRejectsBogusLengthPrefix guards against a single-byte corruption +// of the (uncovered by CRC) length prefix causing an oversized allocation +// (e.g. 0xFFFFFFFF ~ 4GB) instead of a graceful ErrCorrupt. +func TestReaderRejectsBogusLengthPrefix(t *testing.T) { + path := filepath.Join(t.TempDir(), "s.artx") + writeRecords(t, path, []Record{{UUID: [16]byte{1}, Queue: "orders", DrainedAt: 10, AMQP: []byte("hello")}}) + + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + // The 4-byte length prefix of the first record starts right after the + // 5-byte header (offset 5). Corrupt it to an implausibly large value. + binary.BigEndian.PutUint32(b[5:9], 0xFFFFFFFF) + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatal(err) + } + + r, err := OpenReader(path) + if err != nil { + t.Fatal(err) + } + defer r.Close() + + done := make(chan struct{}) + var nextErr error + go func() { + _, _, nextErr = r.Next() + close(done) + }() + + select { + case <-done: + if !errors.Is(nextErr, ErrCorrupt) { + t.Fatalf("expected ErrCorrupt, got %v", nextErr) + } + case <-time.After(5 * time.Second): + t.Fatal("Next() did not return promptly; likely attempted an oversized allocation") + } +} diff --git a/internal/store/store_extra_test.go b/internal/store/store_extra_test.go new file mode 100644 index 0000000..0400292 --- /dev/null +++ b/internal/store/store_extra_test.go @@ -0,0 +1,241 @@ +// internal/store/store_extra_test.go +package store + +import ( + "encoding/binary" + "errors" + "hash/crc32" + "io" + "os" + "path/filepath" + "testing" +) + +func TestWriterOffsetAdvances(t *testing.T) { + path := filepath.Join(t.TempDir(), "s.artx") + w, err := NewWriter(path) + if err != nil { + t.Fatal(err) + } + defer w.Close() + if got := w.Offset(); got != headerLen { + t.Fatalf("fresh Offset = %d, want %d", got, headerLen) + } + rec := Record{UUID: [16]byte{1}, Queue: "orders", DrainedAt: 1, AMQP: []byte("hi")} + if err := w.Append(rec); err != nil { + t.Fatal(err) + } + wantBody := 1 + 16 + 8 + 2 + len("orders") + 4 + len("hi") // +1 for the v2 Kind byte + if got, want := w.Offset(), int64(headerLen)+int64(4+wantBody+4); got != want { + t.Fatalf("Offset after append = %d, want %d", got, want) + } +} + +// TestReaderSeekToResumes writes two records, reads the first to learn the +// resume offset, then opens a fresh reader, SeekTo's that offset, and confirms +// it reads only the second record — the redelivery-resume path. +func TestReaderSeekToResumes(t *testing.T) { + path := filepath.Join(t.TempDir(), "s.artx") + writeRecords(t, path, []Record{ + {UUID: [16]byte{1}, Queue: "a", DrainedAt: 1, AMQP: []byte("first")}, + {UUID: [16]byte{2}, Queue: "b", DrainedAt: 2, AMQP: []byte("second")}, + }) + + r1, err := OpenReader(path) + if err != nil { + t.Fatal(err) + } + _, afterFirst, err := r1.Next() + if err != nil { + t.Fatalf("read first: %v", err) + } + r1.Close() + + r2, err := OpenReader(path) + if err != nil { + t.Fatal(err) + } + defer r2.Close() + if err := r2.SeekTo(afterFirst); err != nil { + t.Fatalf("seek: %v", err) + } + rec, _, err := r2.Next() + if err != nil { + t.Fatalf("read after seek: %v", err) + } + if string(rec.AMQP) != "second" { + t.Fatalf("after seek got %q, want second", rec.AMQP) + } + if _, _, err := r2.Next(); !errors.Is(err, io.EOF) { + t.Fatalf("want EOF after last record, got %v", err) + } +} + +// TestSeekToClampsBelowHeader: an offset inside the header is clamped up to the +// first record so a bogus checkpoint can't land mid-header. +func TestSeekToClampsBelowHeader(t *testing.T) { + path := filepath.Join(t.TempDir(), "s.artx") + writeRecords(t, path, []Record{{UUID: [16]byte{1}, Queue: "a", DrainedAt: 1, AMQP: []byte("x")}}) + r, err := OpenReader(path) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if err := r.SeekTo(0); err != nil { + t.Fatalf("seek: %v", err) + } + if rec, _, err := r.Next(); err != nil || string(rec.AMQP) != "x" { + t.Fatalf("clamped seek read = %+v, %v", rec, err) + } +} + +func TestOpenReaderErrors(t *testing.T) { + dir := t.TempDir() + + if _, err := OpenReader(filepath.Join(dir, "missing.artx")); err == nil { + t.Fatal("open missing file: want error") + } + + short := filepath.Join(dir, "short.artx") + if err := os.WriteFile(short, []byte("AR"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := OpenReader(short); err == nil { + t.Fatal("open truncated header: want error") + } + + badMagic := filepath.Join(dir, "badmagic.artx") + if err := os.WriteFile(badMagic, []byte("XXXX\x01"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := OpenReader(badMagic); err == nil { + t.Fatal("open bad magic: want error") + } + + badVer := filepath.Join(dir, "badver.artx") + if err := os.WriteFile(badVer, append([]byte(Magic), 0xFF), 0o600); err != nil { + t.Fatal(err) + } + if _, err := OpenReader(badVer); err == nil { + t.Fatal("open bad version: want error") + } +} + +func TestNewWriterStatError(t *testing.T) { + // A path whose parent is a regular file (not a directory) makes Stat fail + // with ENOTDIR — not IsNotExist — so NewWriter must surface it, not proceed. + dir := t.TempDir() + file := filepath.Join(dir, "afile") + if err := os.WriteFile(file, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := NewWriter(filepath.Join(file, "under.artx")); err == nil { + t.Fatal("want stat error for path under a regular file") + } +} + +func TestNewWriterOpenError(t *testing.T) { + // Parent directory does not exist: Stat reports IsNotExist (allowed), then + // OpenFile fails — the open-store error branch. + sp := filepath.Join(t.TempDir(), "no-such-dir", "s.artx") + if _, err := NewWriter(sp); err == nil { + t.Fatal("want open error for a nonexistent parent directory") + } +} + +func TestSyncAfterCloseErrors(t *testing.T) { + path := filepath.Join(t.TempDir(), "s.artx") + w, err := NewWriter(path) + if err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatalf("close: %v", err) + } + // fsync on the already-closed file must error rather than silently succeed. + if err := w.Sync(); err == nil { + t.Fatal("want error syncing a closed writer") + } +} + +func TestLoadCheckpointReadError(t *testing.T) { + // A checkpoint path that is a directory makes ReadFile fail with a non- + // IsNotExist error, which LoadCheckpoint must propagate. + sp := filepath.Join(t.TempDir(), "s.artx") + if err := os.Mkdir(ckptPath(sp), 0o700); err != nil { + t.Fatal(err) + } + if _, err := LoadCheckpoint(sp); err == nil { + t.Fatal("want read error when checkpoint path is a directory") + } +} + +func TestLoadCheckpointBadContent(t *testing.T) { + sp := filepath.Join(t.TempDir(), "s.artx") + if err := os.WriteFile(ckptPath(sp), []byte("not-a-number"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadCheckpoint(sp); err == nil { + t.Fatal("want parse error on non-numeric checkpoint") + } +} + +func TestSaveCheckpointErrorsOnBadDir(t *testing.T) { + // A store path whose parent directory does not exist can't be written. + sp := filepath.Join(t.TempDir(), "no-such-dir", "s.artx") + if err := SaveCheckpoint(sp, 1); err == nil { + t.Fatal("want error saving into a nonexistent directory") + } +} + +// frameRecord builds a store file whose single record has a valid length prefix +// and CRC but an arbitrary body, so Next passes CRC validation and exercises the +// decodeBody consistency checks. +func frameRecord(t *testing.T, body []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "framed.artx") + buf := append([]byte(Magic), Version) + var lenPfx [4]byte + binary.BigEndian.PutUint32(lenPfx[:], uint32(len(body))) + buf = append(buf, lenPfx[:]...) + buf = append(buf, body...) + var crc [4]byte + binary.BigEndian.PutUint32(crc[:], crc32.ChecksumIEEE(body)) + buf = append(buf, crc[:]...) + if err := os.WriteFile(path, buf, 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestDecodeBodyRejectsInconsistentRecords(t *testing.T) { + cases := map[string][]byte{ + // < 16+8+2 bytes: too short to hold even the fixed header. + "too short": make([]byte, 10), + // 26 bytes with queue length 0xFFFF: the queue can't fit in the body. + "queue overflow": func() []byte { + b := make([]byte, 26) + binary.BigEndian.PutUint16(b[24:26], 0xFFFF) + return b + }(), + // qlen=0 but amqp length 5 with no trailing bytes: amqp can't fit. + "amqp overflow": func() []byte { + b := make([]byte, 30) + binary.BigEndian.PutUint32(b[26:30], 5) + return b + }(), + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + path := frameRecord(t, body) + r, err := OpenReader(path) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if _, _, err := r.Next(); !errors.Is(err, ErrCorrupt) { + t.Fatalf("want ErrCorrupt, got %v", err) + } + }) + } +} diff --git a/internal/store/writer.go b/internal/store/writer.go new file mode 100644 index 0000000..69a6748 --- /dev/null +++ b/internal/store/writer.go @@ -0,0 +1,116 @@ +package store + +import ( + "bufio" + "encoding/binary" + "fmt" + "hash/crc32" + "math" + "os" +) + +// Writer appends records to a WAL store file. It buffers writes and tracks the +// current byte offset; call Sync to make appended records durable and Close +// when done. A Writer is not safe for concurrent use. +type Writer struct { + f *os.File + buf *bufio.Writer + offset int64 +} + +// NewWriter opens path for appending and writes the file header if the file is +// new or empty. It refuses to open an existing non-empty store, so re-running a +// drain after a crash cannot silently overwrite already-drained records. +func NewWriter(path string) (*Writer, error) { + // Refuse to overwrite an existing non-empty store. Re-running + // `export --out X` after a crash is the natural recovery action; with + // O_TRUNC that would permanently wipe already-drained-and-acked records + // (data loss). A zero-length or missing file is safe to (re)initialize. + if fi, err := os.Stat(path); err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("store %s already exists and is non-empty; choose a new path to avoid overwriting drained data", path) + } else if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("stat store: %w", err) + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + return nil, fmt.Errorf("open store: %w", err) + } + w := &Writer{f: f, buf: bufio.NewWriter(f)} + if _, err := w.buf.WriteString(Magic); err != nil { + return nil, err + } + if err := w.buf.WriteByte(Version); err != nil { + return nil, err + } + w.offset = headerLen + return w, nil +} + +// Append encodes r and writes it to the buffered stream: a length prefix, the +// record body (UUID, drain time, queue, AMQP bytes), and a CRC32 over the body. +// It does not flush; the record is not durable until Sync. It errors if the +// queue name or payload exceeds the format's length limits. +func (w *Writer) Append(r Record) error { + if len(r.Queue) > math.MaxUint16 { + return fmt.Errorf("store: queue name too long: %d bytes (max 65535)", len(r.Queue)) + } + // Payload is the AMQP bytes (KindAMQP) or the serialized Core payload + // (KindCore); only one is populated per record. + payload := r.AMQP + if r.Kind == KindCore { + payload = r.CorePayload + } + if len(payload) > math.MaxUint32 { + return fmt.Errorf("store: payload too long: %d bytes", len(payload)) + } + + body := make([]byte, 0, 1+16+8+2+len(r.Queue)+4+len(payload)) + body = append(body, r.Kind) + body = append(body, r.UUID[:]...) + body = binary.BigEndian.AppendUint64(body, uint64(r.DrainedAt)) + body = binary.BigEndian.AppendUint16(body, uint16(len(r.Queue))) + body = append(body, r.Queue...) + body = binary.BigEndian.AppendUint32(body, uint32(len(payload))) + body = append(body, payload...) + + var hdr [4]byte + binary.BigEndian.PutUint32(hdr[:], uint32(len(body))) + crc := crc32.ChecksumIEEE(body) + + if _, err := w.buf.Write(hdr[:]); err != nil { + return err + } + if _, err := w.buf.Write(body); err != nil { + return err + } + if err := binary.Write(w.buf, binary.BigEndian, crc); err != nil { + return err + } + w.offset += int64(4 + len(body) + 4) + return nil +} + +// Sync flushes the buffer and fsyncs the file, making every appended record +// durable on disk. The drain loop calls this before acking messages on the +// broker so a crash never loses an acked message. +func (w *Writer) Sync() error { + if err := w.buf.Flush(); err != nil { + return err + } + return w.f.Sync() +} + +// Offset returns the byte offset just past the last appended record — the point +// a reader would resume from. It reflects buffered (not necessarily synced) +// writes. +func (w *Writer) Offset() int64 { return w.offset } + +// Close flushes any buffered records and closes the file. It does not fsync; +// call Sync first if the final records must be durable. +func (w *Writer) Close() error { + if err := w.buf.Flush(); err != nil { + _ = w.f.Close() + return err + } + return w.f.Close() +} diff --git a/internal/store/writer_test.go b/internal/store/writer_test.go new file mode 100644 index 0000000..d8a7c05 --- /dev/null +++ b/internal/store/writer_test.go @@ -0,0 +1,230 @@ +// internal/store/writer_test.go +package store + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestWriterWritesHeaderAndRecord(t *testing.T) { + path := filepath.Join(t.TempDir(), "s.artx") + w, err := NewWriter(path) + if err != nil { + t.Fatalf("new writer: %v", err) + } + rec := Record{UUID: [16]byte{1, 2, 3}, Queue: "orders", DrainedAt: 42, AMQP: []byte("hello")} + if err := w.Append(rec); err != nil { + t.Fatalf("append: %v", err) + } + if err := w.Sync(); err != nil { + t.Fatalf("sync: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close: %v", err) + } + b, _ := os.ReadFile(path) + if string(b[:4]) != Magic { + t.Fatalf("missing magic, got %q", b[:4]) + } + if b[4] != Version { + t.Fatalf("bad version %d", b[4]) + } + if len(b) <= 5 { + t.Fatalf("record not written") + } +} + +func TestWriterEncodesRecordBytesExactly(t *testing.T) { + path := filepath.Join(t.TempDir(), "s.artx") + w, err := NewWriter(path) + if err != nil { + t.Fatalf("new writer: %v", err) + } + + wantUUID := [16]byte{0xde, 0xad, 0xbe, 0xef, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c} + wantQueue := "orders" + wantDrainedAt := int64(42) + wantAMQP := []byte("hello") + + rec := Record{UUID: wantUUID, Queue: wantQueue, DrainedAt: wantDrainedAt, AMQP: wantAMQP} + if err := w.Append(rec); err != nil { + t.Fatalf("append: %v", err) + } + if err := w.Sync(); err != nil { + t.Fatalf("sync: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read file: %v", err) + } + + if string(b[:4]) != Magic { + t.Fatalf("bad magic: got %q", b[:4]) + } + if b[4] != Version { + t.Fatalf("bad version: got %d", b[4]) + } + + // v2 body leads with a 1-byte Kind, then the v1 layout. + wantBodyLen := 1 + 16 + 8 + 2 + len(wantQueue) + 4 + len(wantAMQP) + + totalLen := binary.BigEndian.Uint32(b[5:9]) + if int(totalLen) != wantBodyLen { + t.Fatalf("bad totalLen: got %d, want %d", totalLen, wantBodyLen) + } + + fullBody := b[9 : 9+int(totalLen)] + + if fullBody[0] != KindAMQP { + t.Fatalf("bad kind: got %d, want %d", fullBody[0], KindAMQP) + } + body := fullBody[1:] + + gotUUID := body[0:16] + if !bytes.Equal(gotUUID, wantUUID[:]) { + t.Fatalf("bad uuid: got %x, want %x", gotUUID, wantUUID) + } + + gotDrainedAt := int64(binary.BigEndian.Uint64(body[16:24])) + if gotDrainedAt != wantDrainedAt { + t.Fatalf("bad drainedAt: got %d, want %d", gotDrainedAt, wantDrainedAt) + } + + queueLen := binary.BigEndian.Uint16(body[24:26]) + if int(queueLen) != len(wantQueue) { + t.Fatalf("bad queueLen: got %d, want %d", queueLen, len(wantQueue)) + } + + queueStart := 26 + queueEnd := queueStart + int(queueLen) + gotQueue := string(body[queueStart:queueEnd]) + if gotQueue != wantQueue { + t.Fatalf("bad queue: got %q, want %q", gotQueue, wantQueue) + } + + amqpLenStart := queueEnd + amqpLenEnd := amqpLenStart + 4 + amqpLen := binary.BigEndian.Uint32(body[amqpLenStart:amqpLenEnd]) + if int(amqpLen) != len(wantAMQP) { + t.Fatalf("bad amqpLen: got %d, want %d", amqpLen, len(wantAMQP)) + } + + amqpStart := amqpLenEnd + amqpEnd := amqpStart + int(amqpLen) + gotAMQP := body[amqpStart:amqpEnd] + if !bytes.Equal(gotAMQP, wantAMQP) { + t.Fatalf("bad amqp: got %q, want %q", gotAMQP, wantAMQP) + } + + crcStart := 9 + int(totalLen) + crcEnd := crcStart + 4 + gotCRC := binary.BigEndian.Uint32(b[crcStart:crcEnd]) + wantCRC := crc32.ChecksumIEEE(fullBody) + if gotCRC != wantCRC { + t.Fatalf("bad crc: got %d, want %d", gotCRC, wantCRC) + } + + wantFileSize := 5 + 4 + int(totalLen) + 4 + if len(b) != wantFileSize { + t.Fatalf("bad file size: got %d, want %d", len(b), wantFileSize) + } +} + +// TestNewWriterRefusesNonEmptyExisting is the C2 regression: re-running +// `export --out X` after a crash must not truncate an already-populated store +// (permanent loss of drained-and-acked records). NewWriter must error and +// leave the file byte-for-byte intact. +func TestNewWriterRefusesNonEmptyExisting(t *testing.T) { + path := filepath.Join(t.TempDir(), "existing.artx") + original := []byte("ARTX\x01already-drained-records-here") + if err := os.WriteFile(path, original, 0o600); err != nil { + t.Fatalf("seed file: %v", err) + } + before, err := os.Stat(path) + if err != nil { + t.Fatalf("stat before: %v", err) + } + + w, err := NewWriter(path) + if err == nil { + _ = w.Close() + t.Fatal("want error opening a non-empty existing store, got nil") + } + + after, err := os.Stat(path) + if err != nil { + t.Fatalf("stat after: %v", err) + } + if after.Size() != before.Size() { + t.Fatalf("store was truncated: size %d -> %d", before.Size(), after.Size()) + } + got, _ := os.ReadFile(path) + if !bytes.Equal(got, original) { + t.Fatalf("store contents modified: got %q want %q", got, original) + } +} + +// TestNewWriterAcceptsFreshAndZeroLength verifies NewWriter succeeds and writes +// the header for a missing path and for a pre-existing zero-length file (both +// safe to initialize). +func TestNewWriterAcceptsFreshAndZeroLength(t *testing.T) { + t.Run("fresh path", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "fresh.artx") + w, err := NewWriter(path) + if err != nil { + t.Fatalf("new writer on fresh path: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close: %v", err) + } + b, _ := os.ReadFile(path) + if len(b) != headerLen || string(b[:4]) != Magic || b[4] != Version { + t.Fatalf("header not written correctly: %q", b) + } + }) + t.Run("zero-length existing", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "empty.artx") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatalf("seed empty: %v", err) + } + w, err := NewWriter(path) + if err != nil { + t.Fatalf("new writer on zero-length file: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close: %v", err) + } + b, _ := os.ReadFile(path) + if len(b) != headerLen || string(b[:4]) != Magic || b[4] != Version { + t.Fatalf("header not written correctly: %q", b) + } + }) +} + +func TestAppendRejectsOverlongQueueName(t *testing.T) { + path := filepath.Join(t.TempDir(), "s.artx") + w, err := NewWriter(path) + if err != nil { + t.Fatalf("new writer: %v", err) + } + defer w.Close() + + rec := Record{ + UUID: [16]byte{1, 2, 3}, + Queue: strings.Repeat("q", 65536), + DrainedAt: 1, + AMQP: []byte("x"), + } + if err := w.Append(rec); err == nil { + t.Fatalf("expected error for over-length queue name, got nil") + } +} diff --git a/main.go b/main.go deleted file mode 100644 index a7f08c6..0000000 --- a/main.go +++ /dev/null @@ -1,361 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "log" - "os" - "os/signal" - "strconv" - "strings" - "sync" - "sync/atomic" - "syscall" - "time" - - "github.com/go-stomp/stomp" - "github.com/google/uuid" -) - -type collectionVar []string - -func (c *collectionVar) String() string { - return fmt.Sprintf("%v", *c) -} - -func (c *collectionVar) Set(value string) error { - *c = append(*c, value) - return nil -} - -const ( - aqMsgTimestamp = "timestamp" - aqMsgID = "messageID" - aqMsgTraceID = "TRACE_ID" - aqMsgSCS = "SCS" -) - -var ( - serverAddr string - serverUser string - serverPass string - queueName string - method string - messages collectionVar - bodyType string - filePath string - msgTraceID string - msgID string - apiName string - startTime time.Time - startTimeStr string - endTime time.Time - endTimeStr string - helpFlag = false - - closing = make(chan struct{}) -) - -func init() { - flag.StringVar(&serverAddr, "b", "localhost:61613", "broker URL") - flag.StringVar(&serverUser, "u", "", "username") - flag.StringVar(&serverPass, "p", "", "password") - flag.StringVar(&queueName, "q", "", "Destination queue") - flag.StringVar(&method, "m", "C", "Method type. 'C' is consuming, 'P' is producing") - flag.StringVar(&msgTraceID, "trace-id", "", "Filter for trace id") - flag.StringVar(&msgID, "message-id", "", "Filter for message id") - flag.StringVar(&apiName, "api", "", "Filter for api name") - flag.StringVar(&startTimeStr, "startdate", "", "start time for filtering messages in RFC3339 format. e.g.: 2024-01-01T15:05:05Z") - flag.StringVar(&endTimeStr, "enddate", "", "end time for filtering messages in RFC3339 format. e.g.: 2024-01-01T15:05:05Z") - flag.Var(&messages, "message", "Message to send") - flag.StringVar(&bodyType, "t", "plain", "Type of message body. 'plain'=text/plain, 'json'=application/json") - flag.StringVar(&filePath, "f", "", "Given file name to save data") - flag.BoolVar(&helpFlag, "help", false, "Print help text") - flag.Parse() - - var err error - - if helpFlag || len(os.Args) == 1 { - fmt.Fprintf(os.Stderr, "Usage of %s\n", os.Args[0]) - flag.PrintDefaults() - os.Exit(1) - } - - if serverUser == "" { - log.Fatalln("username must be provided!") - } - - if serverPass == "" { - log.Fatalln("password must be provided!") - } - - if queueName == "" { - log.Fatalln("queue must be provided!") - } - - if startTimeStr != "" { - startTime, err = time.Parse(time.RFC3339, startTimeStr) - if err != nil { - log.Fatalf("Invalid start time format: %v\n", err) - } - } - - if endTimeStr != "" { - endTime, err = time.Parse(time.RFC3339, endTimeStr) - if err != nil { - log.Fatalf("Invalid end time format: %v\n", err) - } - } - -} - -func main() { - conn, err := stomp.Dial("tcp", serverAddr, - stomp.ConnOpt.Login(serverUser, serverPass), - stomp.ConnOpt.Host(strings.Split(serverAddr, ":")[0])) - if err != nil { - log.Fatalln("cannot connect to server", err) - } - defer conn.Disconnect() - - var wg sync.WaitGroup - - go func() { - signals := make(chan os.Signal, 1) - signal.Notify(signals, os.Interrupt, syscall.SIGTERM) - <-signals - log.Println("Initiating shutdown of consumer...") - close(closing) - }() - - switch method { - case "C": - wg.Add(1) - go consumeMessages(conn, &wg) - case "P": - wg.Add(1) - go sendMessages(conn, &wg) - default: - log.Fatalln("unknown method has been provided!") - } - - wg.Wait() - -} - -func consumeMessages(conn *stomp.Conn, wg *sync.WaitGroup) { - defer wg.Done() - - var counter atomic.Int32 - - intoFile := filePath != "" - - var file *os.File - defer file.Close() - if intoFile { - file = createFile(filePath) - } - - sub, err := conn.Subscribe(queueName, stomp.AckClientIndividual, stomp.SubscribeOpt.Header("consumer-window-size", "-1")) - if err != nil { - log.Fatalf("cannot subscribe to %s: %v\n", queueName, err) - } - defer sub.Unsubscribe() - - log.Println("Waiting for messages...") - - lastMessageTime := time.Now() - -loop: - for { - select { - case <-closing: - break loop - case msg := <-sub.C: - counter.Add(1) - - if msg.Err != nil { - log.Println("Error during consuming messages:", msg.Err.Error()) - continue - } - - if msg == nil { - fmt.Println("Subscription is already closed.") - break loop - } - - // Filtering messages. - if skipMessage(msg) { - continue - } - - lastMessageTime = time.Now() - - if intoFile { - _, err := file.WriteString(string(msg.Body) + "\n") - if err != nil { - log.Printf("Failed to write message to file: %v\n", err) - } - } else { - fmt.Printf("%s - %s\n", msg.Header.Get(aqMsgTraceID), string(msg.Body)) - } - - if intoFile && counter.Load()%1000 == 0 { - fmt.Printf("%d messages processed so far...\n", counter.Load()) - } - case <-time.After(5 * time.Second): - // Check if no message has been consumed for the last 5 seconds - // Only do echo if writing into file. - if intoFile && time.Since(lastMessageTime) > 5*time.Second { - log.Println("Waiting for messages...") - lastMessageTime = time.Now() - } - } - } - - log.Println("Finished consuming the messages! Total messages: ", counter.Load()) -} - -// skipMessage is a filter. -// If a given filter is not equal with the actual value, -// then the message will be skipped. -func skipMessage(msg *stomp.Message) bool { - - // Date interval filter - if !startTime.IsZero() || !endTime.IsZero() { - timestampStr := msg.Header.Get(aqMsgTimestamp) - if timestampStr != "" { - timeMillis, err := strconv.ParseInt(timestampStr, 10, 64) - if err != nil { - log.Printf("id:%s - invalid timestamp header\n", msg.Header.Get(aqMsgID)) - return true - } - - // If start time is bigger then the actual timestamp then, - // skip the message. - if !startTime.IsZero() { - if timeMillis < startTime.UnixMilli() { - return true - } - } - - // If the end time is lower the the actual timestamp then, - // skip the message. - if !endTime.IsZero() { - if timeMillis > endTime.UnixMilli() { - return true - } - } - - } else { - log.Printf("id:%s - timestamp has been not found in header!\n", msg.Header.Get(aqMsgID)) - return true - } - } - - // Trace ID filter - if msgTraceID != "" { - id := msg.Header.Get(aqMsgTraceID) - if msgTraceID == "" { - log.Printf("id:%s - %s property is missing\n", aqMsgTraceID, msg.Header.Get(aqMsgTraceID)) - return true - } - - if msgTraceID != id { - return true - } - } - - // Message ID filter - if msgID != "" { - id := msg.Header.Get(aqMsgID) - if msgID == "" { - log.Printf("id:%s - %s property is missing\n", aqMsgTraceID, msg.Header.Get(aqMsgID)) - return true - } - - if msgID != id { - return true - } - } - // API name filter - if apiName != "" { - msgAPI := msg.Header.Get(aqMsgSCS) - if msgAPI == "" { - log.Printf("id:%s - %s property is missing\n", aqMsgSCS, msg.Header.Get(aqMsgSCS)) - return true - } - - if msgAPI != apiName { - return true - } - } - - return false -} - -func sendMessages(conn *stomp.Conn, wg *sync.WaitGroup) { - defer wg.Done() - - var err error - var counter atomic.Int32 - - if len(messages) == 0 { - log.Fatalln("No message was provided to send!") - return - } - - isJSON := false - var messageBodyType string - switch bodyType { - case "plain": - messageBodyType = "text/plain" - case "json": - messageBodyType = "application/json" - default: - log.Fatalln("Invalid message body type was provided:", bodyType) - } - - for _, msg := range messages { - var data []byte - if isJSON { - // process json message - data, err = json.Marshal(&msg) - if err != nil { - log.Printf("Failed to marshal message to JSON. Skipping...: %v\n", err) - continue - } - } else { - // process plain text message - data = []byte(msg) - } - - messageID := uuid.NewString() - - // Send json message - err = conn.Send( - queueName, - messageBodyType, - data, - stomp.SendOpt.Header("persistent", "true"), - stomp.SendOpt.Header("messageId", messageID)) - if err != nil { - log.Println("Failed to send message:", err) - } - - counter.Add(1) - } - - log.Println("Finished sending messages! Total sent: ", counter.Load()) -} - -func createFile(path string) *os.File { - // Open the file to write messages - file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - log.Fatalf("Failed to open file for writing: %v\n", err) - } - - return file -}