diff --git a/.github/workflows/build-distribution-kit.yml b/.github/workflows/build-distribution-kit.yml new file mode 100644 index 0000000..6fe9405 --- /dev/null +++ b/.github/workflows/build-distribution-kit.yml @@ -0,0 +1,68 @@ +name: Build reviewed distribution kit + +on: + workflow_dispatch: + inputs: + theme: + description: Native carousel theme + required: true + default: intelligence-dark + type: choice + options: + - intelligence-dark + - editorial-light + +permissions: + contents: read + +concurrency: + group: distribution-kit-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Check out published data + uses: actions/checkout@v7 + with: + ref: gh-pages + path: published + + - uses: actions/setup-go@v6 + with: + go-version: "1.25.12" + cache: true + + - name: Validate the published evidence bundle + run: go run ./cmd/validator -dir published/data -min-reporters 40 + + - name: Build the review-pending kit + run: | + go run ./cmd/distributor \ + -briefing published/data/briefing.json \ + -out "${RUNNER_TEMP}/distribution-kit" \ + -base-url https://elecpapaya.github.io/TradeGravity/ \ + -theme "${{ inputs.theme }}" + + - name: Verify upload assets remain review-gated + shell: bash + run: | + test -f "${RUNNER_TEMP}/distribution-kit/carousel/slide-01.png" + test -f "${RUNNER_TEMP}/distribution-kit/carousel/slide-06.png" + test -f "${RUNNER_TEMP}/distribution-kit/carousel/caption.md" + grep -q '"distribution_status": "review_pending"' "${RUNNER_TEMP}/distribution-kit/manifest.json" + grep -q '"social_publish_authorized": false' "${RUNNER_TEMP}/distribution-kit/manifest.json" + grep -q '"theme": "${{ inputs.theme }}"' "${RUNNER_TEMP}/distribution-kit/manifest.json" + grep -q '"caption_path": "carousel/caption.md"' "${RUNNER_TEMP}/distribution-kit/manifest.json" + + - name: Upload review artifact + uses: actions/upload-artifact@v4 + with: + name: tradegravity-distribution-kit-${{ github.run_id }} + path: ${{ runner.temp }}/distribution-kit + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 2eddbd3..76fd2f5 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -45,11 +45,43 @@ jobs: - name: Validate offline sample dataset run: go run ./cmd/validator -dir examples/sample-data -min-reporters 3 + - name: Build offline distribution kit + run: | + go run ./cmd/distributor \ + -briefing examples/sample-data/briefing.json \ + -out "${RUNNER_TEMP}/distribution-kit" \ + -base-url https://example.org/TradeGravity/ + test -f "${RUNNER_TEMP}/distribution-kit/email/body.html" + grep -q '{{UNSUBSCRIBE_URL}}' "${RUNNER_TEMP}/distribution-kit/email/body.html" + test -f "${RUNNER_TEMP}/distribution-kit/carousel/slide-06.svg" + test -f "${RUNNER_TEMP}/distribution-kit/carousel/slide-06.png" + test -f "${RUNNER_TEMP}/distribution-kit/carousel/caption.md" + grep -q 'https://example.org/TradeGravity/?tab=semiconductors' "${RUNNER_TEMP}/distribution-kit/carousel/caption.md" + test -f "${RUNNER_TEMP}/distribution-kit/manifest.json" + grep -q '"theme": "intelligence-dark"' "${RUNNER_TEMP}/distribution-kit/manifest.json" + grep -q '"caption_path": "carousel/caption.md"' "${RUNNER_TEMP}/distribution-kit/manifest.json" + go run ./cmd/distribution-approval \ + -kit "${RUNNER_TEMP}/distribution-kit" \ + -reviewer ci-synthetic-reviewer \ + -audience synthetic-internal-pilot \ + -channels email,instagram \ + -approved-at 2026-07-17T12:00:00Z \ + -attest-reviewed + test -f "${RUNNER_TEMP}/distribution-kit/approval.json" + grep -q '"provider_delivery_ready": false' "${RUNNER_TEMP}/distribution-kit/approval.json" + go run ./cmd/instagram-preflight \ + -kit "${RUNNER_TEMP}/distribution-kit" \ + -out "${RUNNER_TEMP}/instagram-preflight.json" \ + -generated-at 2026-07-17T13:00:00Z + grep -q '"manual_upload_required": true' "${RUNNER_TEMP}/instagram-preflight.json" + grep -q '"automatic_publish_authorized": false' "${RUNNER_TEMP}/instagram-preflight.json" + grep -q '"contains_credentials": false' "${RUNNER_TEMP}/instagram-preflight.json" + - name: Check browser JavaScript - run: node --check site/app.js && node --check site/security.js && node --check site/data-tools.js && node --check site/explorer-tools.js && node --check site/intelligence-tools.js && node --check site/semiconductor-tools.js && node --check site/experience-tools.js && node --check site/news-tools.js + run: node --check site/app.js && node --check site/security.js && node --check site/data-tools.js && node --check site/explorer-tools.js && node --check site/intelligence-tools.js && node --check site/semiconductor-tools.js && node --check site/experience-tools.js && node --check site/news-tools.js && node --check site/briefing-tools.js - name: Test browser data helpers - run: node --test site/security.test.cjs site/data-tools.test.cjs site/explorer-tools.test.cjs site/intelligence-tools.test.cjs site/semiconductor-tools.test.cjs site/experience-tools.test.cjs site/news-tools.test.cjs site/structure.test.cjs + run: node --test site/security.test.cjs site/data-tools.test.cjs site/explorer-tools.test.cjs site/intelligence-tools.test.cjs site/semiconductor-tools.test.cjs site/experience-tools.test.cjs site/news-tools.test.cjs site/briefing-tools.test.cjs site/structure.test.cjs - name: Validate citation metadata run: | diff --git a/.gitignore b/.gitignore index 312d1f3..f080e75 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,13 @@ /site/data/ tradegravity.db + +/private/ +/distribution-kit/ +/delivery-preflight.json +/instagram-preflight.json +/email-launch-authorization.json +subscriptions.db +subscriptions.db-* +delivery-ledger.db +delivery-ledger.db-* diff --git a/CHANGELOG.md b/CHANGELOG.md index c41676f..332f8e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,16 @@ All notable changes to TradeGravity will be documented in this file. The project - Global PNG snapshot, CSV, and Markdown summary-report exports for the active analysis view. - Always-visible metric, observation-period, scope, and limitation context plus a definitions dialog. - Current, partial, and degraded publication-health states with recovery guidance. +- A validated `briefing.json` contract that derives three cited semiconductor observations and exposes review-gated email Markdown and 4:5 social-carousel copy without collecting subscribers or publishing automatically. +- An offline distribution-kit CLI and manual read-only Actions workflow that render one-primary-CTA email HTML with an unsubscribe placeholder, six matched 1080×1350 SVG originals and PNG upload assets, alt text, approval gates, and deterministic file hashes without sending or posting. +- A second original `editorial-light` native carousel theme behind the same validated renderer interface; theme choice is recorded in the manifest and therefore bound to editorial approval, with no browser runtime, remote font, image fetch, or arbitrary HTML input. +- Automated palette contrast checks across every gradient stop, including normal text and large bold role labels; the light theme's muted and accent colors were tightened to retain the documented floor. +- A review-pending Instagram `caption.md` derived from the same three validated signals, with comparison period, evidence link, conservative scope note, restrained tags, an editorial length ceiling, and manifest/approval tamper protection. +- An Instagram manual-publish preflight CLI that requires channel approval, verifies every PNG plus caption and alt-text contracts, emits content-free aggregate evidence outside the kit, and explicitly carries no credentials or publish authorization. +- A content-release approval CLI that rejects changed, missing, or untracked kit files and binds the verified manifest to a reviewer, audience label, time, and email/Instagram channel set without claiming delivery readiness. +- A local email preflight CLI that validates double opt-in, audience identity, suppression precedence, timestamps, duplicate addresses, unique opaque HTTPS unsubscribe URLs, and a pilot ceiling while emitting an aggregate plan with no recipient addresses or tokens and no delivery authorization. +- A private SQLite subscription-registry CLI and subscription HTTP service with a default-off double-opt-in signup form, short-lived purpose-separated HMAC confirmation links, read-only scanner-safe GET plus explicit confirmation POST, stable Resend confirmation idempotency, form-encoded one-click unsubscribe, signed raw-body provider feedback, durable global suppression, security headers, and private preflight exports. +- A short-lived email launch-approval contract and Resend pilot CLI that replay consent/suppression checks at send time, render recipient-specific visible and RFC one-click unsubscribe links, use one recipient and one provider idempotency key per request, and prevent automatic duplicate or uncertain retries with a private HMAC-keyed SQLite delivery ledger. A separate reconciliation CLI records provider-confirmed acceptance or non-acceptance without storing recipient PII; only the latter plus a different launch-authorization digest permits retry. ### Changed diff --git a/README.md b/README.md index fb4629b..a46f7ba 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,11 @@ TradeGravity is an open-source pipeline and static intelligence dashboard for un - **Live demo:** https://elecpapaya.github.io/TradeGravity/ - **System design:** [DESIGN.md](DESIGN.md) - **Published data schema:** [docs/DATA_SCHEMA.md](docs/DATA_SCHEMA.md) +- **Reviewed distribution workflow:** [docs/DISTRIBUTION.md](docs/DISTRIBUTION.md) +- **Email consent and suppression preflight:** [docs/EMAIL_DELIVERY_PREFLIGHT.md](docs/EMAIL_DELIVERY_PREFLIGHT.md) +- **Provider-backed email pilot:** [docs/EMAIL_PROVIDER_PILOT.md](docs/EMAIL_PROVIDER_PILOT.md) +- **Private registry and unsubscribe service:** [docs/UNSUBSCRIBE_SERVICE.md](docs/UNSUBSCRIBE_SERVICE.md) +- **Instagram manual-publish preflight:** [docs/INSTAGRAM_PREFLIGHT.md](docs/INSTAGRAM_PREFLIGHT.md) - **Semiconductor atlas methodology:** [docs/SEMICONDUCTOR_ATLAS.md](docs/SEMICONDUCTOR_ATLAS.md) - **Reuse examples:** [docs/USAGE.md](docs/USAGE.md) - **Data rights and attribution:** [docs/DATA_RIGHTS.md](docs/DATA_RIGHTS.md) @@ -70,6 +75,13 @@ The pipeline refresh timestamp indicates when TradeGravity generated the site; i - HS2 product mix for the selected reporter, kept separate from WITS headline totals. - Shareable Overview, US–China Lens, Chip Lens, Products, Data & Quality, and Scenario Lab tabs with synchronized filters, country, semiconductor stage/context, product, tariff, and scenario-assumption state. - A semiconductor Pulse that separates latest month-to-month movement from publish-to-publish coverage and value revisions, with a machine-readable bounded change feed. +- A deterministic `briefing.json` distribution draft with three cited semiconductor observations, review-gated email Markdown, and review-gated 4:5 carousel copy. The static site does not collect subscribers, send email, or publish to social platforms. +- An offline `cmd/distributor` build that turns a ready briefing into email HTML/Markdown, a cited Instagram caption, alt text, and six matched 1080×1350 SVG/PNG cards in either `intelligence-dark` or `editorial-light`; all assets share review gates and deterministic hashes without making a network request. +- An aggregate-only `cmd/instagram-preflight` that requires an unchanged Instagram approval, decodes all six PNGs, validates caption evidence/scope/tags and six alt-text sections, refuses output inside the kit, and keeps credentials and automatic publishing explicitly false. +- A fail-closed `cmd/distribution-approval` step that verifies the complete file set and SHA-256 manifest before recording a channel-specific content approval; provider delivery, subscriber consent, and automatic publishing remain explicitly false. +- A local `cmd/distribution-preflight` gate that validates private double-opt-in and suppression CSVs, approved-audience identity, and unique opaque HTTPS unsubscribe URLs, enforces a pilot ceiling, and writes only aggregate counts and digests—never recipient addresses or tokens—while keeping provider configuration and delivery authorization false. +- A separate SQLite `cmd/subscription-registry` and bounded `cmd/unsubscribe-service` that can collect double-opt-in consent outside the static dashboard, send short-lived Resend confirmations, activate only on explicit confirmation POST, issue HMAC-authenticated links without email/audience claims, keep link-scanner GETs read-only, record idempotent RFC one-click suppressions, verify signed Resend feedback, and export private preflight inputs. +- A fail-closed Resend pilot path that binds a one-hour launch approval to the exact aggregate preflight, sender, audience, and content/input digests; reruns the consent and suppression checks immediately before delivery; adds visible and header one-click unsubscribe links; isolates every recipient in a separate provider request; and records only HMAC recipient keys in a private SQLite ledger. Accepted or uncertain attempts are never sent again automatically; provider-confirmed non-acceptance still requires a recorded reconciliation and a new launch approval. - Two-anchor position metrics whose formulas are visible: USA share, China share, exposure balance, position shift, dual exposure, and anchor-growth divergence. - Unadjusted bilateral mirror-reporting diagnostics that compare both countries' reports without choosing either as ground truth or treating the difference as fraud, evasion, rerouting, or an adjusted estimate. - An illustrative HS6 tariff sensitivity lab that can load a published MFN rate and product import baseline while exposing elasticity, pass-through, fallback, and source assumptions. @@ -141,6 +153,7 @@ The public deployment exposes stable machine-readable endpoints: - `https://elecpapaya.github.io/TradeGravity/data/semiconductors/reference.json` - `https://elecpapaya.github.io/TradeGravity/data/semiconductors/monthly/index.json` - `https://elecpapaya.github.io/TradeGravity/data/changes.json` +- `https://elecpapaya.github.io/TradeGravity/data/briefing.json` - `https://elecpapaya.github.io/TradeGravity/data/tariffs/index.json` - `https://elecpapaya.github.io/TradeGravity/data/bilateral-matrix/index.json` - `https://elecpapaya.github.io/TradeGravity/data/mirror/index.json` @@ -276,13 +289,13 @@ This repository reads operating-system environment variables and does not load a ## Generated files and deployment - Local SQLite database: `tradegravity.db` -- Published JSON: `meta.json`, `catalog.json`, `changes.json`, `latest.json`, `series.json`, `quality.json`, `context.json`, `products/`, `strategic-hs6/`, `semiconductors/reference.json`, `semiconductors/monthly/`, `tariffs/`, `bilateral-matrix/`, `mirror/`, and `explanations/` under `site/data/` +- Published JSON: `meta.json`, `catalog.json`, `changes.json`, `briefing.json`, `latest.json`, `series.json`, `quality.json`, `context.json`, `products/`, `strategic-hs6/`, `semiconductors/reference.json`, `semiconductors/monthly/`, `tariffs/`, `bilateral-matrix/`, `mirror/`, and `explanations/` under `site/data/` Generated data and the local database are intentionally not committed to the default branch. The scheduled or manually dispatched core workflow runs the broad collectors and saves its validated database as a three-day Actions artifact. The staggered semiconductor workflow restores that artifact and the previous `gh-pages` publication, adds annual and monthly chip observations for [`configs/chip_connectors.csv`](configs/chip_connectors.csv), emits a validated publish-to-publish `changes.json`, and deploys `site/` to the `gh-pages` branch. A `main` push uses the latest validated `data/` directory from `gh-pages` and redeploys the site without calling WITS, UN Comtrade, WITS/TRAINS, or World Bank APIs. This keeps code-only deployments fast while the weekly refresh remains the source of new published observations. The fast deployment intentionally fails if `gh-pages` does not contain `data/latest.json` and `data/meta.json`. Bootstrap or repair the published dataset by manually running **Update TradeGravity core**, then **Update TradeGravity semiconductor**; the second workflow waits out any remaining quota window before it publishes. -Before deployment, `cmd/validator` checks provenance across every artifact, reporter uniqueness, periods, non-negative finite values, totals and shares, matrix availability/count identities, tariff rate identities, product keys, bounded publication-change arithmetic and ordering, context coverage, and explanation evidence references. +Before deployment, `cmd/validator` checks provenance across every artifact, reporter uniqueness, periods, non-negative finite values, totals and shares, matrix availability/count identities, tariff rate identities, product keys, bounded publication-change arithmetic and ordering, briefing arithmetic and mandatory human-review gates, context coverage, and explanation evidence references. ## Maintenance and contributing diff --git a/ROADMAP.md b/ROADMAP.md index ad3f1f1..0729f00 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -38,6 +38,13 @@ TradeGravity is maintained as a small, inspectable public-data pipeline with an - Publish USA/China exposure balance, position shift, and dual-exposure metrics with explicit formulas and non-alignment caveats across the Intelligence and Chip Lens views. - Collect and publish a focused 12-month, 30-code semiconductor turning-point panel for selected connector economies. - Publish a validated semiconductor Pulse with latest-month movement, previous-publication coverage/value changes, a bounded machine-readable change feed, and evidence endpoints in the Markdown report. +- Publish a deterministic, validator-gated semiconductor distribution brief with cited email Markdown and 4:5 carousel copy; require human review and keep delivery and subscriber storage disabled in the static site. +- Generate a deterministic offline review kit with two network-free native carousel themes, one-primary-CTA email HTML/Markdown, a cited and scope-limited Instagram caption, six matched 1080×1350 SVG/PNG assets, alt text, an editorial/privacy checklist, and a theme/caption-bound SHA-256 manifest; expose it through a read-only manual Actions workflow. +- Require an aggregate Instagram preflight after content approval to revalidate manifest integrity, six upload-ready PNG dimensions, caption evidence/scope/tags, and alt-text completeness while retaining manual upload and false automatic-publish authorization. +- Bind each reviewed kit to an immutable manifest digest, reviewer, non-sensitive audience label, explicit time, and approved channel set while keeping consent, provider-delivery, and automatic-publish readiness false. +- Validate private double-opt-in and suppression CSVs plus unique opaque HTTPS unsubscribe URLs against an email-approved kit, enforce a bounded pilot audience, and emit an address/token-free aggregate preflight plan with provider and delivery authorization still false. +- Maintain consent and suppression in a private SQLite registry; provide an opt-in signup form with short-lived, purpose-separated HMAC confirmation links and stable Resend idempotency; require explicit confirmation POST; issue PII-free unsubscribe links; and verify/deduplicate signed Resend suppression feedback behind an HTTPS reverse proxy. +- Provide a fail-closed Resend pilot adapter with short-lived launch approval, live preflight replay, per-recipient requests, RFC one-click headers, provider idempotency keys, a PII-free local delivery ledger, and explicit provider-evidence reconciliation. Require a new launch approval after confirmed non-acceptance and never automatically retry an unresolved request; keep credentials, subscriber inputs, authorization, and the ledger outside the repository. - Publish unadjusted mirror-reporting diagnostics against USA and China counterpart reports without selecting a ground truth or claiming fraud, rerouting, or reconciliation. - Register only free/public semiconductor evidence layers, including OECD ICIO as lagged industry context, and validate that paid/proprietary sources cannot become required metric inputs. @@ -48,6 +55,7 @@ TradeGravity is maintained as a small, inspectable public-data pipeline with an - Run the documented task with at least three students, researchers, or developers and publish consented findings through [the public study tracker](https://github.com/elecpapaya/TradeGravity/issues/3). - Track task completion, interpretation errors, and time-to-answer in [`docs/USER_TESTING.md`](docs/USER_TESTING.md); never substitute synthetic sessions for real participants. - Publish and maintain the reproducible ASEAN example notebook. +- Verify the deployed `briefing.json`, then run a small editorial review that checks whether readers can trace every distributed observation back to its period, value, and evidence artifact. ## Next: usability and analytical depth @@ -62,11 +70,13 @@ TradeGravity is maintained as a small, inspectable public-data pipeline with an - Prototype ESI/ECI/ICI/SPDI/RPI only after formulas, benchmark datasets, uncertainty, and validation cases are documented. - Add tariff-change decomposition and Marimekko/waterfall views when the required multi-year product coverage is reliable. - [Add versioned scenario manifests and reproducible result artifacts](https://github.com/elecpapaya/TradeGravity/issues/13) before introducing SMART-like substitution or welfare outputs. +- Run the first provider-backed email pilot only after configuring Resend and an authenticated sender domain, deploying signup/confirmation/unsubscribe/signed-feedback endpoints over HTTPS with proxy rate limits, proving feedback replay and restore operations, and obtaining a fresh launch approval for a deliberately confirmed audience. The adapter exists, but no production send is implied. Do not place provider credentials or subscriber data in the static site. ## Later: provider resilience and interpretation - Add configurable provider fallback without mixing provenance silently. - Extend the implemented semiconductor publication-change feed to headline, tariff, and matrix artifacts after each layer has a stable comparison key and revision policy. - Expand grounded explanations after external evaluation confirms they reduce rather than increase interpretation errors. +- Render branded Instagram/LinkedIn card images from the reviewed carousel contract only after an accessibility, citation legibility, platform-rights, and editorial-approval review; keep direct social publishing as a separate, revocable step. Priorities may change when upstream APIs change or users report higher-impact needs. Roadmap discussion should happen in a GitHub issue so decisions remain public and reviewable. diff --git a/cmd/distribution-approval/main.go b/cmd/distribution-approval/main.go new file mode 100644 index 0000000..c764178 --- /dev/null +++ b/cmd/distribution-approval/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/distributionkit" +) + +func main() { + kitDir := flag.String("kit", "distribution-kit", "reviewed distribution-kit directory") + reviewer := flag.String("reviewer", "", "human reviewer name or stable handle; do not include subscriber data") + audience := flag.String("audience", "", "non-sensitive audience label, for example internal-pilot") + channelsValue := flag.String("channels", "", "comma-separated content channels: email,instagram") + approvedAtValue := flag.String("approved-at", "", "explicit RFC3339 approval time") + attested := flag.Bool("attest-reviewed", false, "confirm evidence, copy, rights, assets, and alt text were reviewed") + flag.Parse() + + approvedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*approvedAtValue)) + if err != nil { + fatal(fmt.Errorf("approved-at must be RFC3339: %w", err)) + } + channels := splitChannels(*channelsValue) + approval, content, err := distributionkit.Approve(*kitDir, distributionkit.ApprovalRequest{ + Reviewer: *reviewer, + Audience: *audience, + Channels: channels, + ApprovedAt: approvedAt, + Attested: *attested, + }) + if err != nil { + fatal(err) + } + if err := distributionkit.WriteApproval(*kitDir, content); err != nil { + fatal(err) + } + fmt.Printf("content approval recorded (edition=%s channels=%s scope=%s delivery_ready=%t)\n", approval.EditionID, strings.Join(approval.Channels, ","), approval.Scope, approval.ProviderDeliveryReady) +} + +func splitChannels(value string) []string { + if strings.TrimSpace(value) == "" { + return nil + } + parts := strings.Split(value, ",") + for index := range parts { + parts[index] = strings.TrimSpace(parts[index]) + } + return parts +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "distribution approval failed:", err) + os.Exit(1) +} diff --git a/cmd/distribution-preflight/main.go b/cmd/distribution-preflight/main.go new file mode 100644 index 0000000..5246ca3 --- /dev/null +++ b/cmd/distribution-preflight/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/deliverypreflight" +) + +func main() { + kitDir := flag.String("kit", "distribution-kit", "approved distribution-kit directory") + subscribers := flag.String("subscribers", "", "local double-opt-in subscriber CSV outside the kit") + suppressions := flag.String("suppressions", "", "local unsubscribe/bounce/complaint suppression CSV outside the kit") + output := flag.String("out", "delivery-preflight.json", "new aggregate-only preflight JSON outside the kit") + generatedAtValue := flag.String("generated-at", "", "explicit RFC3339 preflight time") + maxRecipients := flag.Int("max-recipients", 25, "fail when the consented unsuppressed pilot audience exceeds this limit (1-1000)") + flag.Parse() + + generatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*generatedAtValue)) + if err != nil { + fatal(fmt.Errorf("generated-at must be RFC3339: %w", err)) + } + result, err := deliverypreflight.Build(deliverypreflight.Request{ + KitDir: *kitDir, + SubscriberCSV: *subscribers, + SuppressionCSV: *suppressions, + GeneratedAt: generatedAt, + MaxRecipients: *maxRecipients, + }) + if err != nil { + fatal(err) + } + if err := deliverypreflight.Write(*output, *kitDir, result.JSON); err != nil { + fatal(err) + } + fmt.Printf("email preflight passed (edition=%s audience=%s consented=%d suppressed=%d eligible=%d delivery_authorized=%t)\n", result.Plan.EditionID, result.Plan.Audience, result.Plan.Counts.Consented, result.Plan.Counts.Suppressed, result.Plan.Counts.Eligible, result.Plan.DeliveryAuthorized) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "distribution preflight failed:", err) + os.Exit(1) +} diff --git a/cmd/distributor/main.go b/cmd/distributor/main.go new file mode 100644 index 0000000..fe2cdd7 --- /dev/null +++ b/cmd/distributor/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "tradegravity/internal/distributionkit" +) + +func main() { + briefingPath := flag.String("briefing", "site/data/briefing.json", "review-gated briefing.json input") + outputDir := flag.String("out", "distribution-kit", "new output directory; existing paths are never overwritten") + baseURL := flag.String("base-url", "https://elecpapaya.github.io/TradeGravity/", "public evidence base URL") + theme := flag.String("theme", distributionkit.ThemeIntelligenceDark, "carousel theme: intelligence-dark or editorial-light") + flag.Parse() + + raw, err := os.ReadFile(*briefingPath) + if err != nil { + fatal(fmt.Errorf("read briefing: %w", err)) + } + bundle, err := distributionkit.BuildWithOptions(raw, *baseURL, distributionkit.BuildOptions{Theme: *theme}) + if err != nil { + fatal(err) + } + if err := distributionkit.Write(*outputDir, bundle); err != nil { + fatal(err) + } + fmt.Printf("distribution kit built (edition=%s theme=%s files=%d status=%s out=%s)\n", bundle.Manifest.EditionID, bundle.Manifest.Carousel.Theme, len(bundle.Files), bundle.Manifest.DistributionStatus, *outputDir) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "distribution kit failed:", err) + os.Exit(1) +} diff --git a/cmd/email-delivery-reconcile/main.go b/cmd/email-delivery-reconcile/main.go new file mode 100644 index 0000000..e399673 --- /dev/null +++ b/cmd/email-delivery-reconcile/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/emaildelivery" +) + +const ledgerSecretEnvironment = "TRADEGRAVITY_DELIVERY_SECRET" + +func main() { + ledgerPath := flag.String("ledger", "", "private SQLite delivery ledger") + edition := flag.String("edition", "", "edition ID from the stopped delivery") + audience := flag.String("audience", "", "non-sensitive audience label") + recipient := flag.String("recipient", "", "private recipient address used only to derive the ledger key") + outcome := flag.String("outcome", "", "provider-confirmed outcome: accepted or not_accepted") + providerMessageID := flag.String("provider-message-id", "", "required only for an accepted outcome") + resolvedBy := flag.String("resolved-by", "", "operator identity recorded in the private ledger") + evidence := flag.String("evidence", "", "non-sensitive provider-dashboard or support evidence label") + resolvedAtValue := flag.String("resolved-at", "", "explicit RFC3339 reconciliation time") + attest := flag.Bool("attest-provider-checked", false, "attest that the provider dashboard or support record was checked") + flag.Parse() + + if !*attest { + fatal(fmt.Errorf("attest-provider-checked is required")) + } + resolvedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*resolvedAtValue)) + if err != nil { + fatal(fmt.Errorf("resolved-at must be RFC3339: %w", err)) + } + ledger, err := emaildelivery.OpenLedger(*ledgerPath, []byte(os.Getenv(ledgerSecretEnvironment))) + if err != nil { + fatal(err) + } + defer ledger.Close() + result, err := ledger.Reconcile(context.Background(), emaildelivery.ReconciliationRequest{ + EditionID: *edition, + Audience: *audience, + Email: *recipient, + Outcome: strings.TrimSpace(*outcome), + ProviderMessageID: *providerMessageID, + ResolvedBy: *resolvedBy, + Evidence: *evidence, + ResolvedAt: resolvedAt, + }) + if err != nil { + fatal(err) + } + fmt.Printf("delivery reconciliation recorded (edition=%s audience=%s outcome=%s changed=%t already_resolved=%t)\n", *edition, *audience, *outcome, result.Changed, result.AlreadyResolved) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "email delivery reconciliation failed:", err) + os.Exit(1) +} diff --git a/cmd/email-delivery/main.go b/cmd/email-delivery/main.go new file mode 100644 index 0000000..c39dbfc --- /dev/null +++ b/cmd/email-delivery/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/emaildelivery" +) + +const ( + apiKeyEnvironment = "RESEND_API_KEY" + ledgerSecretEnvironment = "TRADEGRAVITY_DELIVERY_SECRET" +) + +func main() { + kitDir := flag.String("kit", "distribution-kit", "approved distribution-kit directory") + subscribers := flag.String("subscribers", "", "private double-opt-in subscriber CSV") + suppressions := flag.String("suppressions", "", "private suppression CSV") + preflight := flag.String("preflight", "delivery-preflight.json", "aggregate preflight bound to the launch authorization") + authorization := flag.String("authorization", "email-launch-authorization.json", "short-lived private launch authorization") + ledger := flag.String("ledger", "", "private SQLite delivery ledger") + sendAtValue := flag.String("send-at", "", "explicit RFC3339 send time inside the authorization window") + sendLive := flag.Bool("send-live", false, "make live provider requests; required in addition to all other gates") + flag.Parse() + + sendAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*sendAtValue)) + if err != nil { + fatal(fmt.Errorf("send-at must be RFC3339: %w", err)) + } + provider, err := emaildelivery.NewResendProvider(os.Getenv(apiKeyEnvironment), nil) + if err != nil { + fatal(err) + } + result, err := emaildelivery.Deliver(context.Background(), emaildelivery.DeliveryRequest{ + KitDir: *kitDir, + SubscriberCSV: *subscribers, + SuppressionCSV: *suppressions, + PreflightPath: *preflight, + AuthorizationPath: *authorization, + LedgerPath: *ledger, + LedgerSecret: []byte(os.Getenv(ledgerSecretEnvironment)), + SendAt: sendAt, + Provider: provider, + SendLive: *sendLive, + }) + if err != nil { + fatal(err) + } + fmt.Printf("email delivery complete (edition=%s audience=%s eligible=%d accepted=%d skipped=%d pending=%d)\n", result.EditionID, result.Audience, result.Eligible, result.Accepted, result.Skipped, result.Pending) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "email delivery failed:", err) + os.Exit(1) +} diff --git a/cmd/email-launch-approval/main.go b/cmd/email-launch-approval/main.go new file mode 100644 index 0000000..377bb9e --- /dev/null +++ b/cmd/email-launch-approval/main.go @@ -0,0 +1,63 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/emaildelivery" +) + +func main() { + preflight := flag.String("preflight", "delivery-preflight.json", "aggregate delivery preflight JSON") + output := flag.String("out", "email-launch-authorization.json", "new private launch authorization JSON") + provider := flag.String("provider", "resend", "email provider for this pilot authorization") + sender := flag.String("from", "", "authenticated sender mailbox") + replyTo := flag.String("reply-to", "", "optional reply-to mailbox") + authorizedBy := flag.String("authorized-by", "", "operator identity recorded in the launch authorization") + authorizedAtValue := flag.String("authorized-at", "", "explicit RFC3339 authorization time") + expiresAtValue := flag.String("expires-at", "", "explicit RFC3339 expiry no more than one hour later") + domainReady := flag.Bool("attest-domain-authenticated", false, "attest that SPF, DKIM, and DMARC were verified for the sender") + feedbackReady := flag.Bool("attest-feedback-ready", false, "attest that bounce and complaint handling is operational") + privacyReady := flag.Bool("attest-privacy-reviewed", false, "attest that retention, deletion, and operator access were reviewed") + pilotReady := flag.Bool("attest-pilot-recipients", false, "attest that every eligible pilot recipient was intentionally confirmed") + flag.Parse() + + authorizedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*authorizedAtValue)) + if err != nil { + fatal(fmt.Errorf("authorized-at must be RFC3339: %w", err)) + } + expiresAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*expiresAtValue)) + if err != nil { + fatal(fmt.Errorf("expires-at must be RFC3339: %w", err)) + } + authorization, raw, err := emaildelivery.Authorize(emaildelivery.AuthorizationRequest{ + PreflightPath: *preflight, + Provider: *provider, + Sender: *sender, + ReplyTo: *replyTo, + AuthorizedBy: *authorizedBy, + AuthorizedAt: authorizedAt, + ExpiresAt: expiresAt, + Attestations: emaildelivery.Attestations{ + SenderDomainAuthenticated: *domainReady, + BounceComplaintReady: *feedbackReady, + PrivacyControlsReviewed: *privacyReady, + PilotRecipientsConfirmed: *pilotReady, + }, + }) + if err != nil { + fatal(err) + } + if err := emaildelivery.WriteAuthorization(*output, raw); err != nil { + fatal(err) + } + fmt.Printf("email launch authorized (edition=%s audience=%s provider=%s eligible=%d expires_at=%s)\n", authorization.EditionID, authorization.Audience, authorization.Provider, authorization.EligibleRecipients, authorization.ExpiresAt) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "email launch approval failed:", err) + os.Exit(1) +} diff --git a/cmd/instagram-preflight/main.go b/cmd/instagram-preflight/main.go new file mode 100644 index 0000000..cf50906 --- /dev/null +++ b/cmd/instagram-preflight/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/socialpreflight" +) + +func main() { + kit := flag.String("kit", "distribution-kit", "Instagram-approved distribution-kit directory") + out := flag.String("out", "instagram-preflight.json", "new aggregate-only JSON outside the kit") + generated := flag.String("generated-at", "", "explicit RFC3339 preflight time") + flag.Parse() + generatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*generated)) + if err != nil { + fatal(fmt.Errorf("generated-at must be RFC3339: %w", err)) + } + result, err := socialpreflight.Build(*kit, generatedAt) + if err != nil { + fatal(err) + } + if err := socialpreflight.Write(*out, *kit, result.JSON); err != nil { + fatal(err) + } + fmt.Printf("Instagram preflight passed (edition=%s theme=%s slides=%d caption_runes=%d hashtags=%d manual_upload=%t automatic_publish=%t)\n", result.Plan.EditionID, result.Plan.Theme, result.Plan.SlideCount, result.Plan.CaptionRunes, result.Plan.HashtagCount, result.Plan.ManualUploadRequired, result.Plan.AutomaticPublishAuthorized) +} + +func fatal(err error) { fmt.Fprintln(os.Stderr, "Instagram preflight failed:", err); os.Exit(1) } diff --git a/cmd/publisher/briefing.go b/cmd/publisher/briefing.go new file mode 100644 index 0000000..2cc463e --- /dev/null +++ b/cmd/publisher/briefing.go @@ -0,0 +1,381 @@ +package main + +import ( + "fmt" + "math" + "sort" + "strings" + "time" +) + +const briefingSchemaVersion = "1.0" + +type briefingFile struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt string `json:"generated_at"` + EditionID string `json:"edition_id"` + Status string `json:"status"` + Title string `json:"title"` + Scope string `json:"scope"` + LatestPeriod string `json:"latest_period,omitempty"` + PreviousPeriod string `json:"previous_period,omitempty"` + PublicationStatus string `json:"publication_status"` + ReviewRequired bool `json:"review_required"` + Signals []briefingSignal `json:"signals"` + Email briefingEmail `json:"email"` + SocialCarousel briefingSocialCarousel `json:"social_carousel"` + Caveats []string `json:"caveats"` + EvidenceEntryPoint string `json:"evidence_entry_point"` +} + +type briefingSignal struct { + ID string `json:"id"` + Kind string `json:"kind"` + Title string `json:"title"` + Summary string `json:"summary"` + ReporterISO3 string `json:"reporter_iso3"` + ReporterName string `json:"reporter_name"` + Classification string `json:"classification,omitempty"` + Code string `json:"code,omitempty"` + Label string `json:"label,omitempty"` + Period string `json:"period"` + PreviousPeriod string `json:"previous_period"` + Current briefingObservedValue `json:"current"` + Previous briefingObservedValue `json:"previous"` + DeltaTradeUSD float64 `json:"delta_trade_usd"` + ChangeRatio *float64 `json:"change_ratio,omitempty"` + ChinaShareDelta float64 `json:"china_share_delta"` + Evidence []string `json:"evidence"` + Interpretation string `json:"interpretation"` + MeasurementLimit string `json:"measurement_limit"` +} + +type briefingObservedValue struct { + USATradeUSD float64 `json:"usa_trade_usd"` + ChinaTradeUSD float64 `json:"china_trade_usd"` + TotalTradeUSD float64 `json:"total_trade_usd"` + ChinaShare float64 `json:"china_share"` +} + +type briefingEmail struct { + Subject string `json:"subject"` + Preview string `json:"preview"` + Markdown string `json:"markdown"` + CTALabel string `json:"cta_label"` + CTAPath string `json:"cta_path"` + SendPolicy string `json:"send_policy"` + PrimaryGoal string `json:"primary_goal"` +} + +type briefingSocialCarousel struct { + Format string `json:"format"` + AspectRatio string `json:"aspect_ratio"` + ReviewPolicy string `json:"review_policy"` + Slides []briefingCarouselSlide `json:"slides"` +} + +type briefingCarouselSlide struct { + Order int `json:"order"` + Role string `json:"role"` + Headline string `json:"headline"` + Body string `json:"body"` + Evidence []string `json:"evidence"` +} + +type briefingCandidate struct { + signal briefingSignal + magnitude float64 +} + +func buildBriefing(generatedAt string, latest []latestEntry, monthlyIndex semiconductorMonthlyIndexFile, monthlyFiles map[string]semiconductorMonthlyFile, publicationChanges publicationChangesFile) briefingFile { + briefing := briefingFile{ + SchemaVersion: briefingSchemaVersion, + GeneratedAt: generatedAt, + EditionID: briefingEditionID(generatedAt, monthlyIndex.Periods), + Status: "unavailable", + Title: "TradeGravity Semiconductor Pulse", + Scope: "Deterministic distribution brief from selected monthly UN Comtrade HS6 observations against USA and China; not a complete semiconductor market, causal claim, or investment recommendation", + PublicationStatus: publicationChanges.Status, + ReviewRequired: true, + Signals: []briefingSignal{}, + EvidenceEntryPoint: "./?tab=semiconductors", + Caveats: []string{ + "Monthly customs observations can be volatile, incomplete, and revised.", + "USA and China values are the two published anchor relationships, not world totals or physical shipment routes.", + "Publication-to-publication revisions are separate from economic month-to-month movement.", + }, + Email: briefingEmail{ + CTALabel: "Inspect the evidence", + CTAPath: "./?tab=semiconductors", + SendPolicy: "manual_review_required", + PrimaryGoal: "Return the reader to the cited TradeGravity evidence", + }, + SocialCarousel: briefingSocialCarousel{ + Format: "carousel_copy", + AspectRatio: "4:5", + ReviewPolicy: "manual_review_required", + Slides: []briefingCarouselSlide{}, + }, + } + if len(monthlyIndex.Periods) > 0 { + briefing.LatestPeriod = monthlyIndex.Periods[len(monthlyIndex.Periods)-1] + } + if len(monthlyIndex.Periods) > 1 { + briefing.PreviousPeriod = monthlyIndex.Periods[len(monthlyIndex.Periods)-2] + } + + names := make(map[string]string, len(latest)) + for _, row := range latest { + name := strings.TrimSpace(row.Name) + if name == "" { + name = row.ISO3 + } + names[strings.ToUpper(row.ISO3)] = name + } + reporterCandidates := make([]briefingCandidate, 0, len(monthlyFiles)) + shareCandidates := make([]briefingCandidate, 0, len(monthlyFiles)) + productCandidates := make([]briefingCandidate, 0) + currentPeriod, previousPeriod := briefing.LatestPeriod, briefing.PreviousPeriod + if currentPeriod == "" || previousPeriod == "" { + briefing.Email.Subject = "TradeGravity Semiconductor Pulse · data unavailable" + briefing.Email.Preview = "The current publication does not contain enough comparable monthly observations to produce a distribution brief." + briefing.Email.Markdown = "# TradeGravity Semiconductor Pulse\n\nNo distribution brief was generated because two comparable monthly observations were not available. This is not interpreted as no change.\n" + return briefing + } + for _, file := range monthlyFiles { + currentAggregate, currentOK := aggregateBriefingRows(file.Rows, currentPeriod) + previousAggregate, previousOK := aggregateBriefingRows(file.Rows, previousPeriod) + if !currentOK || !previousOK { + continue + } + reporter := strings.ToUpper(strings.TrimSpace(file.ReporterISO3)) + name := names[reporter] + if name == "" { + name = reporter + } + reporterSignal := makeBriefingSignal("reporter_total_change", reporter, name, "", "", "", currentPeriod, previousPeriod, currentAggregate, previousAggregate) + reporterSignal.ID = "monthly-total-" + strings.ToLower(reporter) + reporterSignal.Title = fmt.Sprintf("%s selected chip trade %s", name, movementWord(reporterSignal.DeltaTradeUSD)) + reporterSignal.Summary = fmt.Sprintf("Selected monthly HS6 trade with USA and China moved from %s to %s (%s).", formatBriefingUSD(previousAggregate.TotalTradeUSD), formatBriefingUSD(currentAggregate.TotalTradeUSD), formatBriefingPercent(reporterSignal.ChangeRatio)) + reporterSignal.Interpretation = "A change in the selected two-anchor customs observations worth investigating; it does not establish production, demand, or causality." + reporterSignal.Evidence = []string{"./semiconductors/monthly/" + reporter + ".json", "./semiconductors/monthly/index.json"} + reporterCandidates = append(reporterCandidates, briefingCandidate{signal: reporterSignal, magnitude: math.Abs(reporterSignal.DeltaTradeUSD)}) + + shareSignal := reporterSignal + shareSignal.ID = "anchor-share-" + strings.ToLower(reporter) + shareSignal.Kind = "anchor_share_shift" + shareSignal.Title = fmt.Sprintf("%s two-anchor balance shifted %s", name, anchorDirection(shareSignal.ChinaShareDelta)) + shareSignal.Summary = fmt.Sprintf("China's share of the selected USA-plus-China total moved from %.1f%% to %.1f%% (%+.1f percentage points).", previousAggregate.ChinaShare*100, currentAggregate.ChinaShare*100, shareSignal.ChinaShareDelta*100) + shareSignal.Interpretation = "The sign describes movement within the published USA-China anchor pair, not political alignment or global market share." + shareCandidates = append(shareCandidates, briefingCandidate{signal: shareSignal, magnitude: math.Abs(shareSignal.ChinaShareDelta)}) + + currentProducts := briefingProductsByKey(file.Rows, currentPeriod) + previousProducts := briefingProductsByKey(file.Rows, previousPeriod) + for key, current := range currentProducts { + previous, ok := previousProducts[key] + if !ok { + continue + } + productSignal := makeBriefingSignal("product_total_change", reporter, name, current.Classification, current.Code, current.Label, currentPeriod, previousPeriod, observedValueFromMonthly(current), observedValueFromMonthly(previous)) + productSignal.ID = "product-" + strings.ToLower(reporter) + "-" + current.Code + productSignal.Title = fmt.Sprintf("%s · %s %s", name, current.Label, movementWord(productSignal.DeltaTradeUSD)) + productSignal.Summary = fmt.Sprintf("HS6 %s selected trade moved from %s to %s (%s).", current.Code, formatBriefingUSD(productSignal.Previous.TotalTradeUSD), formatBriefingUSD(productSignal.Current.TotalTradeUSD), formatBriefingPercent(productSignal.ChangeRatio)) + productSignal.Interpretation = "This is a product-level customs observation against USA and China, not company revenue, capacity, or a shipment route." + productSignal.Evidence = []string{"./semiconductors/monthly/" + reporter + ".json", "./semiconductors/reference.json"} + productCandidates = append(productCandidates, briefingCandidate{signal: productSignal, magnitude: math.Abs(productSignal.DeltaTradeUSD)}) + } + } + + sortBriefingCandidates(reporterCandidates) + sortBriefingCandidates(shareCandidates) + sortBriefingCandidates(productCandidates) + for _, candidates := range [][]briefingCandidate{reporterCandidates, shareCandidates, productCandidates} { + if len(candidates) > 0 { + briefing.Signals = append(briefing.Signals, candidates[0].signal) + } + } + if len(briefing.Signals) != 3 { + briefing.Email.Subject = "TradeGravity Semiconductor Pulse · data unavailable" + briefing.Email.Preview = "The current publication does not contain enough comparable monthly observations to produce a distribution brief." + briefing.Email.Markdown = "# TradeGravity Semiconductor Pulse\n\nNo distribution brief was generated because two comparable monthly observations were not available. This is not interpreted as no change.\n" + return briefing + } + + briefing.Status = "ready" + briefing.Email = buildBriefingEmail(briefing) + briefing.SocialCarousel = buildBriefingCarousel(briefing) + return briefing +} + +func aggregateBriefingRows(rows []semiconductorMonthlyProductEntry, period string) (briefingObservedValue, bool) { + value := briefingObservedValue{} + found := false + for _, row := range rows { + if row.Period != period { + continue + } + value.USATradeUSD += row.USA.Trade + value.ChinaTradeUSD += row.CHN.Trade + found = true + } + value.TotalTradeUSD = value.USATradeUSD + value.ChinaTradeUSD + if value.TotalTradeUSD > 0 { + value.ChinaShare = value.ChinaTradeUSD / value.TotalTradeUSD + } + return value, found +} + +func briefingProductsByKey(rows []semiconductorMonthlyProductEntry, period string) map[string]semiconductorMonthlyProductEntry { + result := make(map[string]semiconductorMonthlyProductEntry) + for _, row := range rows { + if row.Period != period { + continue + } + key := strings.ToUpper(strings.TrimSpace(row.Classification)) + "|" + row.Code + result[key] = row + } + return result +} + +func observedValueFromMonthly(row semiconductorMonthlyProductEntry) briefingObservedValue { + return briefingObservedValue{USATradeUSD: row.USA.Trade, ChinaTradeUSD: row.CHN.Trade, TotalTradeUSD: row.Total, ChinaShare: row.ShareCN} +} + +func makeBriefingSignal(kind, reporter, name, classification, code, label, period, previousPeriod string, current, previous briefingObservedValue) briefingSignal { + delta := current.TotalTradeUSD - previous.TotalTradeUSD + var ratio *float64 + if previous.TotalTradeUSD > 0 { + value := delta / previous.TotalTradeUSD + ratio = &value + } + return briefingSignal{ + Kind: kind, + ReporterISO3: reporter, + ReporterName: name, + Classification: classification, + Code: code, + Label: label, + Period: period, + PreviousPeriod: previousPeriod, + Current: current, + Previous: previous, + DeltaTradeUSD: delta, + ChangeRatio: ratio, + ChinaShareDelta: current.ChinaShare - previous.ChinaShare, + Evidence: []string{}, + MeasurementLimit: "Selected monthly HS6 observations against USA and China only; subject to source revisions and coverage limits.", + } +} + +func sortBriefingCandidates(candidates []briefingCandidate) { + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].magnitude != candidates[j].magnitude { + return candidates[i].magnitude > candidates[j].magnitude + } + left, right := candidates[i].signal, candidates[j].signal + if left.ReporterISO3 != right.ReporterISO3 { + return left.ReporterISO3 < right.ReporterISO3 + } + return left.Code < right.Code + }) +} + +func buildBriefingEmail(briefing briefingFile) briefingEmail { + lines := []string{ + "# " + briefing.Title, + "", + fmt.Sprintf("Observation window: %s vs %s · publication status: %s", briefing.LatestPeriod, briefing.PreviousPeriod, briefing.PublicationStatus), + "", + } + for _, signal := range briefing.Signals { + lines = append(lines, "## "+signal.Title, "", signal.Summary, "", "Interpretation boundary: "+signal.Interpretation, "") + } + lines = append(lines, + "Review the cited evidence before forwarding or publishing this draft.", + "", + "[Inspect the evidence]({{BASE_URL}}/?tab=semiconductors)", + "", + "Data scope: selected monthly UN Comtrade HS6 observations against USA and China. Not investment, legal, or policy advice.", + ) + return briefingEmail{ + Subject: fmt.Sprintf("TradeGravity Semiconductor Pulse · %s", briefing.LatestPeriod), + Preview: fmt.Sprintf("Three cited USA-China semiconductor observations for %s; monthly movement is kept separate from publication revisions.", briefing.LatestPeriod), + Markdown: strings.Join(lines, "\n"), + CTALabel: "Inspect the evidence", + CTAPath: "./?tab=semiconductors", + SendPolicy: "manual_review_required", + PrimaryGoal: "Return the reader to the cited TradeGravity evidence", + } +} + +func buildBriefingCarousel(briefing briefingFile) briefingSocialCarousel { + slides := []briefingCarouselSlide{{ + Order: 1, Role: "cover", Headline: briefing.Title, + Body: fmt.Sprintf("Three USA-China semiconductor observations · %s vs %s", briefing.LatestPeriod, briefing.PreviousPeriod), + Evidence: []string{"./semiconductors/monthly/index.json"}, + }} + roles := []string{"scale", "anchor_balance", "product"} + for index, signal := range briefing.Signals { + slides = append(slides, briefingCarouselSlide{Order: index + 2, Role: roles[index], Headline: signal.Title, Body: signal.Summary, Evidence: append([]string(nil), signal.Evidence...)}) + } + slides = append(slides, + briefingCarouselSlide{Order: 5, Role: "method", Headline: "Read the clocks separately", Body: "Month-to-month customs movement and publish-to-publish revisions answer different questions. Neither proves causality or a physical route.", Evidence: []string{"./changes.json", "./semiconductors/monthly/index.json"}}, + briefingCarouselSlide{Order: 6, Role: "cta", Headline: "Inspect the evidence", Body: "Open TradeGravity's Chip Lens for periods, values, sources, coverage, and limitations.", Evidence: []string{"./?tab=semiconductors"}}, + ) + return briefingSocialCarousel{Format: "carousel_copy", AspectRatio: "4:5", ReviewPolicy: "manual_review_required", Slides: slides} +} + +func briefingEditionID(generatedAt string, periods []string) string { + period := "no-period" + if len(periods) > 0 { + period = periods[len(periods)-1] + } + timestamp := "unknown" + if parsed, err := time.Parse(time.RFC3339, generatedAt); err == nil { + timestamp = parsed.UTC().Format("20060102T150405Z") + } + return "semiconductor-pulse-" + period + "-" + timestamp +} + +func movementWord(delta float64) string { + if delta > 0 { + return "increased" + } + if delta < 0 { + return "decreased" + } + return "was unchanged" +} + +func anchorDirection(chinaShareDelta float64) string { + if chinaShareDelta > 0 { + return "toward China" + } + if chinaShareDelta < 0 { + return "toward USA" + } + return "without a change" +} + +func formatBriefingUSD(value float64) string { + abs := math.Abs(value) + switch { + case abs >= 1e12: + return fmt.Sprintf("US$%.2fT", value/1e12) + case abs >= 1e9: + return fmt.Sprintf("US$%.2fB", value/1e9) + case abs >= 1e6: + return fmt.Sprintf("US$%.2fM", value/1e6) + case abs >= 1e3: + return fmt.Sprintf("US$%.2fK", value/1e3) + default: + return fmt.Sprintf("US$%.0f", value) + } +} + +func formatBriefingPercent(value *float64) string { + if value == nil { + return "no comparable percentage" + } + return fmt.Sprintf("%+.1f%%", *value*100) +} diff --git a/cmd/publisher/briefing_test.go b/cmd/publisher/briefing_test.go new file mode 100644 index 0000000..d8a1b9c --- /dev/null +++ b/cmd/publisher/briefing_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "strings" + "testing" +) + +func TestBuildBriefingCreatesDeterministicEmailAndCarouselDrafts(t *testing.T) { + generatedAt := "2026-07-17T00:00:00Z" + index := semiconductorMonthlyIndexFile{Periods: []string{"2026-04", "2026-05"}, Reporters: []string{"JPN", "KOR"}} + files := map[string]semiconductorMonthlyFile{ + "KOR.json": briefingMonthlyFile("KOR", []semiconductorMonthlyProductEntry{ + briefingMonthlyRow("2026-04", "854232", "Memories", 100, 100), + briefingMonthlyRow("2026-04", "848620", "Semiconductor manufacturing machinery", 50, 50), + briefingMonthlyRow("2026-05", "854232", "Memories", 180, 120), + briefingMonthlyRow("2026-05", "848620", "Semiconductor manufacturing machinery", 55, 45), + }), + "JPN.json": briefingMonthlyFile("JPN", []semiconductorMonthlyProductEntry{ + briefingMonthlyRow("2026-04", "854232", "Memories", 80, 120), + briefingMonthlyRow("2026-05", "854232", "Memories", 120, 80), + }), + } + latest := []latestEntry{{ISO3: "KOR", Name: "Korea, Rep."}, {ISO3: "JPN", Name: "Japan"}} + changes := publicationChangesFile{Status: "changed"} + + got := buildBriefing(generatedAt, latest, index, files, changes) + if got.Status != "ready" || got.SchemaVersion != "1.0" || got.EditionID != "semiconductor-pulse-2026-05-20260717T000000Z" { + t.Fatalf("unexpected briefing identity: %+v", got) + } + if !got.ReviewRequired || got.PublicationStatus != "changed" || got.LatestPeriod != "2026-05" || got.PreviousPeriod != "2026-04" { + t.Fatalf("unexpected briefing provenance: %+v", got) + } + if len(got.Signals) != 3 { + t.Fatalf("signals = %d, want 3", len(got.Signals)) + } + if got.Signals[0].Kind != "reporter_total_change" || got.Signals[0].ReporterISO3 != "KOR" || got.Signals[0].DeltaTradeUSD != 100 { + t.Fatalf("unexpected scale signal: %+v", got.Signals[0]) + } + if got.Signals[1].Kind != "anchor_share_shift" || got.Signals[1].ReporterISO3 != "JPN" { + t.Fatalf("unexpected anchor signal: %+v", got.Signals[1]) + } + assertFloat(t, "anchor share shift", got.Signals[1].ChinaShareDelta, -0.2) + if got.Signals[2].Kind != "product_total_change" || got.Signals[2].ReporterISO3 != "KOR" || got.Signals[2].Code != "854232" || got.Signals[2].DeltaTradeUSD != 100 { + t.Fatalf("unexpected product signal: %+v", got.Signals[2]) + } + if got.Email.SendPolicy != "manual_review_required" || !strings.Contains(got.Email.Markdown, "{{BASE_URL}}") || !strings.Contains(got.Email.Markdown, got.Signals[0].Title) { + t.Fatalf("email draft does not preserve review and evidence handoff: %+v", got.Email) + } + if got.SocialCarousel.AspectRatio != "4:5" || got.SocialCarousel.ReviewPolicy != "manual_review_required" || len(got.SocialCarousel.Slides) != 6 { + t.Fatalf("unexpected carousel draft: %+v", got.SocialCarousel) + } + for index, slide := range got.SocialCarousel.Slides { + if slide.Order != index+1 || len(slide.Evidence) == 0 { + t.Fatalf("slide %d is not ordered or cited: %+v", index, slide) + } + } +} + +func TestBuildBriefingFailsClosedWithoutTwoComparableMonths(t *testing.T) { + index := semiconductorMonthlyIndexFile{Periods: []string{"2026-05"}, Reporters: []string{"KOR"}} + files := map[string]semiconductorMonthlyFile{ + "KOR.json": briefingMonthlyFile("KOR", []semiconductorMonthlyProductEntry{briefingMonthlyRow("2026-05", "854232", "Memories", 100, 100)}), + } + got := buildBriefing("2026-07-17T00:00:00Z", []latestEntry{{ISO3: "KOR", Name: "Korea, Rep."}}, index, files, publicationChangesFile{Status: "baseline"}) + if got.Status != "unavailable" || len(got.Signals) != 0 || len(got.SocialCarousel.Slides) != 0 { + t.Fatalf("briefing should fail closed: %+v", got) + } + if !strings.Contains(got.Email.Markdown, "not available") || got.Email.SendPolicy != "manual_review_required" { + t.Fatalf("unavailable email state is not explicit: %+v", got.Email) + } +} + +func TestBuildBriefingUsesOnePublicationWindowAcrossReporters(t *testing.T) { + index := semiconductorMonthlyIndexFile{Periods: []string{"2026-03", "2026-04", "2026-05"}, Reporters: []string{"JPN", "KOR"}} + files := map[string]semiconductorMonthlyFile{ + "JPN.json": briefingMonthlyFile("JPN", []semiconductorMonthlyProductEntry{ + briefingMonthlyRow("2026-04", "854232", "Memories", 100, 100), + briefingMonthlyRow("2026-05", "854232", "Memories", 120, 80), + }), + "KOR.json": { + ReporterISO3: "KOR", + Periods: []string{"2026-03", "2026-04"}, + Rows: []semiconductorMonthlyProductEntry{ + briefingMonthlyRow("2026-03", "854232", "Memories", 100, 100), + briefingMonthlyRow("2026-04", "854232", "Memories", 1000, 0), + }, + }, + } + latest := []latestEntry{{ISO3: "JPN", Name: "Japan"}, {ISO3: "KOR", Name: "Korea, Rep."}} + + got := buildBriefing("2026-07-17T00:00:00Z", latest, index, files, publicationChangesFile{Status: "changed"}) + if got.Status != "ready" || got.LatestPeriod != "2026-05" || got.PreviousPeriod != "2026-04" { + t.Fatalf("unexpected common publication window: %+v", got) + } + for _, signal := range got.Signals { + if signal.ReporterISO3 != "JPN" || signal.Period != "2026-05" || signal.PreviousPeriod != "2026-04" { + t.Fatalf("signal escaped the common publication window: %+v", signal) + } + } +} + +func briefingMonthlyFile(reporter string, rows []semiconductorMonthlyProductEntry) semiconductorMonthlyFile { + return semiconductorMonthlyFile{ReporterISO3: reporter, Periods: []string{"2026-04", "2026-05"}, Rows: rows} +} + +func briefingMonthlyRow(period, code, label string, usa, china float64) semiconductorMonthlyProductEntry { + total := usa + china + share := 0.0 + if total > 0 { + share = china / total + } + return semiconductorMonthlyProductEntry{ + Period: period, Classification: "H6", Code: code, Label: label, + USA: seriesBlock{Available: true, Trade: usa}, CHN: seriesBlock{Available: true, Trade: china}, Total: total, ShareCN: share, + } +} diff --git a/cmd/publisher/extended.go b/cmd/publisher/extended.go index 7fc07de..df776ab 100644 --- a/cmd/publisher/extended.go +++ b/cmd/publisher/extended.go @@ -347,7 +347,7 @@ type providerComparison struct { DeltaRatio float64 `json:"delta_ratio"` } -func buildDataCatalog(generatedAt, provider, contextStatus string, series seriesFile, products productIndexFile, strategicIndex strategicIndexFile, tariffIndex tariffIndexFile, matrixIndex matrixIndexFile, mirrorIndex mirrorIndexFile, semiconductorMonthlyIndex semiconductorMonthlyIndexFile, publicationChanges publicationChangesFile, semiconductorReferences ...semiconductor.Reference) dataCatalogFile { +func buildDataCatalog(generatedAt, provider, contextStatus string, series seriesFile, products productIndexFile, strategicIndex strategicIndexFile, tariffIndex tariffIndexFile, matrixIndex matrixIndexFile, mirrorIndex mirrorIndexFile, semiconductorMonthlyIndex semiconductorMonthlyIndexFile, publicationChanges publicationChangesFile, briefing briefingFile, semiconductorReferences ...semiconductor.Reference) dataCatalogFile { semiconductorReference := semiconductor.Reference{} if len(semiconductorReferences) > 0 { semiconductorReference = semiconductorReferences[0] @@ -389,6 +389,10 @@ func buildDataCatalog(generatedAt, provider, contextStatus string, series series if publicationChanges.Status == "changed" || publicationChanges.Status == "unchanged" { publicationChangesStatus = "ready" } + briefingStatus := "partial" + if briefing.Status == "ready" { + briefingStatus = "ready" + } return dataCatalogFile{ SchemaVersion: "1.0", GeneratedAt: generatedAt, @@ -404,6 +408,7 @@ func buildDataCatalog(generatedAt, provider, contextStatus string, series series {ID: "semiconductor_atlas", Title: "Semiconductor value-chain atlas", Status: semiconductorStatus, Provider: "tradegravity + cited official sources", Classification: "stage-mapped source HS revision", ProductLevel: 6, Grain: "stage taxonomy + country role context + policy event + published HS6 coverage", Partitioning: "reference publication + strategic HS6 reporter/year chunks", Href: "./semiconductors/reference.json"}, {ID: "semiconductor_monthly", Title: "Focused US-China semiconductor turning points", Status: semiconductorMonthlyStatus, Provider: semiconductorMonthlyIndex.Provider, Classification: "source HS revision", ProductLevel: 6, Grain: "focused reporter × USA/CHN partner × flow × selected HS6 × month", Partitioning: "index + one file per reporter", Href: "./semiconductors/monthly/index.json"}, {ID: "publication_changes", Title: "Observed publication changes", Status: publicationChangesStatus, Provider: "tradegravity", Classification: "source HS revision", ProductLevel: 6, Grain: "publication × focused reporter × month × selected HS6", Partitioning: "single bounded change feed", Href: "./changes.json"}, + {ID: "distribution_briefing", Title: "Deterministic email and social briefing", Status: briefingStatus, Provider: "tradegravity", Classification: "source HS revision", ProductLevel: 6, Grain: "edition × selected monthly observation × distribution channel", Partitioning: "single reviewed-draft publication", Href: "./briefing.json"}, {ID: "mirror_reconciliation", Title: "Unadjusted mirror-reporting diagnostics", Status: mirrorStatus, Provider: mirrorIndex.Provider, ProductLevel: 0, Grain: "third-country reporter × USA/CHN anchor × mirrored flow × TOTAL × annual period", Partitioning: "index + reporter/year chunks", Href: "./mirror/index.json"}, {ID: "value_added_network", Title: "Value-added supply-chain exposure", Status: "planned", Grain: "origin × destination × industry × year", Partitioning: "year/industry chunks"}, {ID: "scenario_runs", Title: "Versioned scenario outputs", Status: "planned", Grain: "scenario × market × product × partner", Partitioning: "one manifest and result set per run"}, diff --git a/cmd/publisher/extended_test.go b/cmd/publisher/extended_test.go index f4f6f0a..30f880c 100644 --- a/cmd/publisher/extended_test.go +++ b/cmd/publisher/extended_test.go @@ -78,6 +78,7 @@ func TestBuildDataCatalogSeparatesReadyAndPlannedResources(t *testing.T) { mirrorIndexFile{Provider: "comtrade", Partitions: []mirrorPartition{{ReporterISO3: "KOR", Period: "2023"}}}, semiconductorMonthlyIndexFile{Provider: "comtrade", Partitions: []semiconductorMonthlyPartition{{ReporterISO3: "KOR"}}}, publicationChangesFile{Status: "changed"}, + briefingFile{Status: "ready"}, ) if catalog.SchemaVersion != "1.0" || len(catalog.Resources) < 10 { t.Fatalf("unexpected catalog shape: %+v", catalog) @@ -104,6 +105,9 @@ func TestBuildDataCatalogSeparatesReadyAndPlannedResources(t *testing.T) { if byID["publication_changes"].Status != "ready" || byID["publication_changes"].Href != "./changes.json" { t.Fatalf("publication change feed is not published: %+v", byID["publication_changes"]) } + if byID["distribution_briefing"].Status != "ready" || byID["distribution_briefing"].Href != "./briefing.json" { + t.Fatalf("distribution briefing is not published: %+v", byID["distribution_briefing"]) + } } func TestBuildMirrorFilesComparesBothReportedDirectionsWithoutChoosingTruth(t *testing.T) { diff --git a/cmd/publisher/main.go b/cmd/publisher/main.go index 985566f..eae0062 100644 --- a/cmd/publisher/main.go +++ b/cmd/publisher/main.go @@ -234,6 +234,7 @@ func build(args []string) { fmt.Fprintln(os.Stderr, "failed to compare the previous semiconductor publication:", err) os.Exit(1) } + briefing := buildBriefing(now, latest, semiconductorMonthlyIndex, semiconductorMonthlyFiles, publicationChanges) tariffRows, err := loadTariffObservations(*dbPath, "trains") if err != nil { fmt.Fprintln(os.Stderr, "failed to load tariff observations:", err) @@ -253,7 +254,7 @@ func build(args []string) { os.Exit(1) } quality := buildQualityFile(now, *provider, latest, rows, productRows, runs) - catalog := buildDataCatalog(now, *provider, contextData.Status, seriesOutput, productIndex, strategicIndex, tariffIndex, matrixIndex, mirrorIndex, semiconductorMonthlyIndex, publicationChanges, semiconductorReference) + catalog := buildDataCatalog(now, *provider, contextData.Status, seriesOutput, productIndex, strategicIndex, tariffIndex, matrixIndex, mirrorIndex, semiconductorMonthlyIndex, publicationChanges, briefing, semiconductorReference) metadata := buildMeta(now, *provider, partners, rows, latest) augmentMeta(&metadata, latest, seriesOutput, productIndex, len(productRows), contextData.Status) augmentStrategicMeta(&metadata, strategicIndex) @@ -294,6 +295,10 @@ func build(args []string) { fmt.Fprintln(os.Stderr, "failed to write changes.json:", err) os.Exit(1) } + if err := writeJSON(filepath.Join(*outDir, "briefing.json"), briefing); err != nil { + fmt.Fprintln(os.Stderr, "failed to write briefing.json:", err) + os.Exit(1) + } productsDir := filepath.Join(*outDir, "products") if err := os.MkdirAll(productsDir, 0o755); err != nil { fmt.Fprintln(os.Stderr, "failed to create products dir:", err) diff --git a/cmd/subscription-registry/main.go b/cmd/subscription-registry/main.go new file mode 100644 index 0000000..425f079 --- /dev/null +++ b/cmd/subscription-registry/main.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/subscriptions" +) + +const secretEnvironment = "TRADEGRAVITY_UNSUBSCRIBE_SECRET" + +func main() { + databasePath := flag.String("db", "", "private SQLite subscription database path") + publicBaseURL := flag.String("base-url", "", "public HTTPS base URL for the unsubscribe service") + consentPath := flag.String("consents", "", "private active double-opt-in consent CSV") + audience := flag.String("audience", "", "non-sensitive audience label to export") + subscriberOutput := flag.String("out-subscribers", "", "new private delivery subscriber CSV") + suppressionOutput := flag.String("out-suppressions", "", "new private suppression CSV") + importedAtValue := flag.String("imported-at", "", "explicit RFC3339 import time") + flag.Parse() + + importedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*importedAtValue)) + if err != nil { + fatal(fmt.Errorf("imported-at must be RFC3339: %w", err)) + } + secret := []byte(os.Getenv(secretEnvironment)) + if len(secret) == 0 { + fatal(fmt.Errorf("%s is required", secretEnvironment)) + } + consents, err := readPrivateInput(*consentPath) + if err != nil { + fatal(err) + } + registry, err := subscriptions.Open(*databasePath, secret, *publicBaseURL) + if err != nil { + fatal(err) + } + defer registry.Close() + + result, err := registry.ImportConsents(context.Background(), consents, importedAt) + if err != nil { + fatal(err) + } + subscribersCSV, suppressionsCSV, err := registry.ExportAudience(context.Background(), *audience) + if err != nil { + fatal(err) + } + if err := subscriptions.WritePrivateExports(*subscriberOutput, subscribersCSV, *suppressionOutput, suppressionsCSV); err != nil { + fatal(err) + } + fmt.Printf("subscription registry updated (inserted=%d updated=%d suppressed_skipped=%d exports=2)\n", result.Inserted, result.Updated, result.SuppressedSkipped) +} + +func readPrivateInput(path string) ([]byte, error) { + path = strings.TrimSpace(path) + if path == "" { + return nil, errors.New("consent CSV path is required") + } + info, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("inspect consent CSV: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return nil, errors.New("consent CSV must be a regular non-symlink file") + } + if info.Size() > 5<<20 { + return nil, errors.New("consent CSV exceeds the 5 MiB limit") + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read consent CSV: %w", err) + } + return raw, nil +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "subscription registry failed:", err) + os.Exit(1) +} diff --git a/cmd/unsubscribe-service/main.go b/cmd/unsubscribe-service/main.go new file mode 100644 index 0000000..95c509b --- /dev/null +++ b/cmd/unsubscribe-service/main.go @@ -0,0 +1,145 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "html" + "log" + "net/http" + "net/mail" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "tradegravity/internal/emaildelivery" + "tradegravity/internal/subscriptions" +) + +const ( + secretEnvironment = "TRADEGRAVITY_UNSUBSCRIBE_SECRET" + webhookSecretEnvironment = "RESEND_WEBHOOK_SECRET" + resendAPIKeyEnvironment = "RESEND_API_KEY" +) + +type resendConfirmationSender struct { + provider *emaildelivery.ResendProvider + from, replyTo string +} + +func (sender resendConfirmationSender) SendConfirmation(ctx context.Context, confirmation subscriptions.ConfirmationEmail) (string, error) { + safeURL := html.EscapeString(confirmation.ConfirmationURL) + message := emaildelivery.Message{ + From: sender.from, ReplyTo: sender.replyTo, To: confirmation.To, + Subject: "Confirm your TradeGravity briefing subscription", + HTML: `

You requested the TradeGravity research briefing.

Review and confirm your subscription

Opening the link does not subscribe you; use the confirmation button on that page. This link expires soon. If you did not request it, ignore this email.

`, + Text: "You requested the TradeGravity research briefing.\n\nReview and confirm your subscription:\n" + confirmation.ConfirmationURL + "\n\nOpening the link does not subscribe you; use the confirmation button on that page. This link expires soon. If you did not request it, ignore this email.\n", + } + return sender.provider.Send(ctx, message, confirmation.IdempotencyKey) +} + +func main() { + databasePath := flag.String("db", "", "private SQLite subscription database path") + publicBaseURL := flag.String("base-url", "", "public HTTPS base URL for links generated by the registry") + listenAddress := flag.String("listen", "127.0.0.1:8081", "HTTP listen address; use a TLS reverse proxy for public service") + enableResendWebhook := flag.Bool("enable-resend-webhook", false, "enable the signed Resend feedback endpoint") + enableSignup := flag.Bool("enable-signup", false, "enable the public double-opt-in subscription form") + signupAudience := flag.String("signup-audience", "tradegravity-briefing", "fixed audience for the public form") + consentSource := flag.String("consent-source", "public-subscribe-form", "consent evidence source label") + privacyVersion := flag.String("privacy-notice-version", "v1", "privacy notice version recorded with consent") + privacyURL := flag.String("privacy-notice-url", "", "public HTTPS privacy notice URL") + confirmationFrom := flag.String("confirmation-from", "", "verified sender used for confirmation mail") + confirmationReplyTo := flag.String("confirmation-reply-to", "", "optional reply-to address") + confirmationTTL := flag.Duration("confirmation-ttl", 30*time.Minute, "confirmation link lifetime (5m to 24h)") + maximumPending := flag.Int("max-pending", 100, "maximum unexpired pending confirmations") + flag.Parse() + + secret := []byte(os.Getenv(secretEnvironment)) + if len(secret) == 0 { + fatal(fmt.Errorf("%s is required", secretEnvironment)) + } + registry, err := subscriptions.Open(*databasePath, secret, *publicBaseURL) + if err != nil { + fatal(err) + } + defer registry.Close() + + options := subscriptions.HandlerOptions{Now: time.Now} + if *enableResendWebhook { + webhookSecret := os.Getenv(webhookSecretEnvironment) + if strings.TrimSpace(webhookSecret) == "" { + fatal(fmt.Errorf("%s is required when the Resend webhook is enabled", webhookSecretEnvironment)) + } + options.ResendWebhookSecret = webhookSecret + } + if *enableSignup { + provider, providerErr := emaildelivery.NewResendProvider(os.Getenv(resendAPIKeyEnvironment), nil) + if providerErr != nil { + fatal(providerErr) + } + if strings.TrimSpace(*confirmationFrom) == "" { + fatal(errors.New("confirmation-from is required when signup is enabled")) + } + if _, parseErr := mail.ParseAddress(*confirmationFrom); parseErr != nil { + fatal(errors.New("confirmation-from is invalid")) + } + if strings.TrimSpace(*confirmationReplyTo) != "" { + if _, parseErr := mail.ParseAddress(*confirmationReplyTo); parseErr != nil { + fatal(errors.New("confirmation-reply-to is invalid")) + } + } + options.Signup = &subscriptions.SignupOptions{Config: subscriptions.SignupConfig{ + Audience: *signupAudience, ConsentSource: *consentSource, PrivacyNoticeVersion: *privacyVersion, + PrivacyNoticeURL: *privacyURL, ConfirmationTTL: *confirmationTTL, DispatchCooldown: time.Minute, MaxPending: *maximumPending, + }, Sender: resendConfirmationSender{provider: provider, from: *confirmationFrom, replyTo: *confirmationReplyTo}} + } + handler, err := registry.HandlerWithOptions(options) + if err != nil { + fatal(err) + } + server := &http.Server{ + Addr: strings.TrimSpace(*listenAddress), + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 30 * time.Second, + MaxHeaderBytes: 8 << 10, + ErrorLog: log.New(os.Stderr, "unsubscribe service: ", 0), + } + if server.Addr == "" { + fatal(errors.New("listen address is required")) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + done := make(chan error, 1) + go func() { + done <- server.ListenAndServe() + }() + fmt.Printf("subscription service listening (address=%s tls=reverse-proxy-required resend_webhook=%t signup=%t)\n", server.Addr, *enableResendWebhook, *enableSignup) + + select { + case err := <-done: + if !errors.Is(err, http.ErrServerClosed) { + fatal(err) + } + case <-ctx.Done(): + shutdownContext, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := server.Shutdown(shutdownContext); err != nil { + fatal(err) + } + if err := <-done; !errors.Is(err, http.ErrServerClosed) { + fatal(err) + } + } +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "unsubscribe service failed:", err) + os.Exit(1) +} diff --git a/cmd/validator/briefing_test.go b/cmd/validator/briefing_test.go new file mode 100644 index 0000000..56fbbb3 --- /dev/null +++ b/cmd/validator/briefing_test.go @@ -0,0 +1,49 @@ +package main + +import "testing" + +func TestValidateBriefingAcceptsReviewGatedCitedDraft(t *testing.T) { + metadata := datasetMeta{GeneratedAt: "2026-07-17T00:00:00Z"} + briefing := validValidationBriefing(metadata.GeneratedAt) + if err := validateBriefing(metadata, validationPublicationChanges{Status: "changed"}, briefing); err != nil { + t.Fatalf("validateBriefing() error = %v", err) + } +} + +func TestValidateBriefingRejectsAutomaticSendAndBrokenArithmetic(t *testing.T) { + metadata := datasetMeta{GeneratedAt: "2026-07-17T00:00:00Z"} + briefing := validValidationBriefing(metadata.GeneratedAt) + briefing.Email.SendPolicy = "automatic" + if err := validateBriefing(metadata, validationPublicationChanges{Status: "changed"}, briefing); err == nil { + t.Fatal("validateBriefing() accepted an automatic send policy") + } + briefing = validValidationBriefing(metadata.GeneratedAt) + briefing.Signals[0].DeltaTradeUSD = 999 + if err := validateBriefing(metadata, validationPublicationChanges{Status: "changed"}, briefing); err == nil { + t.Fatal("validateBriefing() accepted inconsistent signal arithmetic") + } +} + +func validValidationBriefing(generatedAt string) validationBriefing { + current := validationBriefingObservedValue{USATradeUSD: 120, ChinaTradeUSD: 80, TotalTradeUSD: 200, ChinaShare: 0.4} + previous := validationBriefingObservedValue{USATradeUSD: 80, ChinaTradeUSD: 80, TotalTradeUSD: 160, ChinaShare: 0.5} + ratio := 0.25 + signals := []validationBriefingSignal{ + {ID: "monthly-total-kor", Kind: "reporter_total_change", Title: "Korea total increased", Summary: "Summary", ReporterISO3: "KOR", ReporterName: "Korea, Rep.", Period: "2026-05", PreviousPeriod: "2026-04", Current: current, Previous: previous, DeltaTradeUSD: 40, ChangeRatio: &ratio, ChinaShareDelta: -0.1, Evidence: []string{"./semiconductors/monthly/KOR.json", "./semiconductors/monthly/index.json"}, Interpretation: "Boundary", MeasurementLimit: "Limit"}, + {ID: "anchor-share-kor", Kind: "anchor_share_shift", Title: "Korea share shifted", Summary: "Summary", ReporterISO3: "KOR", ReporterName: "Korea, Rep.", Period: "2026-05", PreviousPeriod: "2026-04", Current: current, Previous: previous, DeltaTradeUSD: 40, ChangeRatio: &ratio, ChinaShareDelta: -0.1, Evidence: []string{"./semiconductors/monthly/KOR.json", "./semiconductors/monthly/index.json"}, Interpretation: "Boundary", MeasurementLimit: "Limit"}, + {ID: "product-kor-854232", Kind: "product_total_change", Title: "Korea memories increased", Summary: "Summary", ReporterISO3: "KOR", ReporterName: "Korea, Rep.", Classification: "H6", Code: "854232", Label: "Memories", Period: "2026-05", PreviousPeriod: "2026-04", Current: current, Previous: previous, DeltaTradeUSD: 40, ChangeRatio: &ratio, ChinaShareDelta: -0.1, Evidence: []string{"./semiconductors/monthly/KOR.json", "./semiconductors/reference.json"}, Interpretation: "Boundary", MeasurementLimit: "Limit"}, + } + roles := []string{"cover", "scale", "anchor_balance", "product", "method", "cta"} + slides := make([]validationBriefingCarouselSlide, 0, len(roles)) + for index, role := range roles { + slides = append(slides, validationBriefingCarouselSlide{Order: index + 1, Role: role, Headline: "Headline", Body: "Body", Evidence: []string{"./semiconductors/monthly/index.json"}}) + } + return validationBriefing{ + SchemaVersion: "1.0", GeneratedAt: generatedAt, EditionID: "semiconductor-pulse-2026-05-20260717T000000Z", Status: "ready", + Title: "TradeGravity Semiconductor Pulse", Scope: "Scope", LatestPeriod: "2026-05", PreviousPeriod: "2026-04", PublicationStatus: "changed", ReviewRequired: true, + Signals: signals, + Email: validationBriefingEmail{Subject: "Subject", Preview: "Preview", Markdown: "[Evidence]({{BASE_URL}})", CTALabel: "Inspect", CTAPath: "./?tab=semiconductors", SendPolicy: "manual_review_required", PrimaryGoal: "Inspect evidence"}, + SocialCarousel: validationBriefingSocialCarousel{Format: "carousel_copy", AspectRatio: "4:5", ReviewPolicy: "manual_review_required", Slides: slides}, + Caveats: []string{"One", "Two", "Three"}, EvidenceEntryPoint: "./?tab=semiconductors", + } +} diff --git a/cmd/validator/extended.go b/cmd/validator/extended.go index ffcae3b..266dedd 100644 --- a/cmd/validator/extended.go +++ b/cmd/validator/extended.go @@ -379,6 +379,78 @@ type validationPublicationRevision struct { ChangeRatio *float64 `json:"change_ratio,omitempty"` } +type validationBriefing struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt string `json:"generated_at"` + EditionID string `json:"edition_id"` + Status string `json:"status"` + Title string `json:"title"` + Scope string `json:"scope"` + LatestPeriod string `json:"latest_period,omitempty"` + PreviousPeriod string `json:"previous_period,omitempty"` + PublicationStatus string `json:"publication_status"` + ReviewRequired bool `json:"review_required"` + Signals []validationBriefingSignal `json:"signals"` + Email validationBriefingEmail `json:"email"` + SocialCarousel validationBriefingSocialCarousel `json:"social_carousel"` + Caveats []string `json:"caveats"` + EvidenceEntryPoint string `json:"evidence_entry_point"` +} + +type validationBriefingSignal struct { + ID string `json:"id"` + Kind string `json:"kind"` + Title string `json:"title"` + Summary string `json:"summary"` + ReporterISO3 string `json:"reporter_iso3"` + ReporterName string `json:"reporter_name"` + Classification string `json:"classification,omitempty"` + Code string `json:"code,omitempty"` + Label string `json:"label,omitempty"` + Period string `json:"period"` + PreviousPeriod string `json:"previous_period"` + Current validationBriefingObservedValue `json:"current"` + Previous validationBriefingObservedValue `json:"previous"` + DeltaTradeUSD float64 `json:"delta_trade_usd"` + ChangeRatio *float64 `json:"change_ratio,omitempty"` + ChinaShareDelta float64 `json:"china_share_delta"` + Evidence []string `json:"evidence"` + Interpretation string `json:"interpretation"` + MeasurementLimit string `json:"measurement_limit"` +} + +type validationBriefingObservedValue struct { + USATradeUSD float64 `json:"usa_trade_usd"` + ChinaTradeUSD float64 `json:"china_trade_usd"` + TotalTradeUSD float64 `json:"total_trade_usd"` + ChinaShare float64 `json:"china_share"` +} + +type validationBriefingEmail struct { + Subject string `json:"subject"` + Preview string `json:"preview"` + Markdown string `json:"markdown"` + CTALabel string `json:"cta_label"` + CTAPath string `json:"cta_path"` + SendPolicy string `json:"send_policy"` + PrimaryGoal string `json:"primary_goal"` +} + +type validationBriefingSocialCarousel struct { + Format string `json:"format"` + AspectRatio string `json:"aspect_ratio"` + ReviewPolicy string `json:"review_policy"` + Slides []validationBriefingCarouselSlide `json:"slides"` +} + +type validationBriefingCarouselSlide struct { + Order int `json:"order"` + Role string `json:"role"` + Headline string `json:"headline"` + Body string `json:"body"` + Evidence []string `json:"evidence"` +} + type validationQuality struct { SchemaVersion string `json:"schema_version"` GeneratedAt string `json:"generated_at"` @@ -550,6 +622,13 @@ func validateExtendedDataset(dataDir string, metadata datasetMeta, latest datase if err := validatePublicationChanges(metadata, semiconductorMonthlyIndex, publicationChanges); err != nil { return err } + var briefing validationBriefing + if err := readJSON(filepath.Join(dataDir, "briefing.json"), &briefing); err != nil { + return fmt.Errorf("read briefing.json: %w", err) + } + if err := validateBriefing(metadata, publicationChanges, briefing); err != nil { + return err + } var tariffIndex validationTariffIndex if err := readJSON(filepath.Join(dataDir, "tariffs", "index.json"), &tariffIndex); err != nil { return fmt.Errorf("read tariff index: %w", err) @@ -575,7 +654,7 @@ func validateExtendedDataset(dataDir string, metadata datasetMeta, latest datase if err := readJSON(filepath.Join(dataDir, "catalog.json"), &catalog); err != nil { return fmt.Errorf("read catalog.json: %w", err) } - if err := validateCatalog(metadata, catalog, publicationChanges); err != nil { + if err := validateCatalog(metadata, catalog, publicationChanges, briefing); err != nil { return err } if err := validateExplanations(dataDir, metadata, latest); err != nil { @@ -588,7 +667,7 @@ func validateExtendedDataset(dataDir string, metadata datasetMeta, latest datase return validateContext(metadata, latest, contextData) } -func validateCatalog(metadata datasetMeta, catalog validationCatalog, publicationChanges validationPublicationChanges) error { +func validateCatalog(metadata datasetMeta, catalog validationCatalog, publicationChanges validationPublicationChanges, briefing validationBriefing) error { if catalog.SchemaVersion != "1.0" || catalog.GeneratedAt != metadata.GeneratedAt { return errorsForExtended("catalog provenance does not match metadata") } @@ -611,7 +690,7 @@ func validateCatalog(metadata datasetMeta, catalog validationCatalog, publicatio } seen[resource.ID] = resource } - for _, required := range []string{"headline_totals", "time_series", "country_context", "product_chapters", "quality", "strategic_hs6", "tariff_schedules", "bilateral_matrix", "semiconductor_atlas", "semiconductor_monthly", "publication_changes", "mirror_reconciliation", "scenario_runs"} { + for _, required := range []string{"headline_totals", "time_series", "country_context", "product_chapters", "quality", "strategic_hs6", "tariff_schedules", "bilateral_matrix", "semiconductor_atlas", "semiconductor_monthly", "publication_changes", "distribution_briefing", "mirror_reconciliation", "scenario_runs"} { if _, ok := seen[required]; !ok { return fmt.Errorf("catalog is missing resource %q", required) } @@ -668,6 +747,14 @@ func validateCatalog(metadata datasetMeta, catalog validationCatalog, publicatio if changesResource.Status != wantChangesStatus || changesResource.Provider != "tradegravity" || changesResource.ProductLevel != 6 || changesResource.Href != "./changes.json" { return errorsForExtended("catalog publication change resource does not match changes.json") } + briefingResource := seen["distribution_briefing"] + wantBriefingStatus := "partial" + if briefing.Status == "ready" { + wantBriefingStatus = "ready" + } + if briefingResource.Status != wantBriefingStatus || briefingResource.Provider != "tradegravity" || briefingResource.ProductLevel != 6 || briefingResource.Href != "./briefing.json" { + return errorsForExtended("catalog distribution briefing resource does not match briefing.json") + } return nil } @@ -741,6 +828,120 @@ func validatePublicationChanges(metadata datasetMeta, monthly validationSemicond return nil } +func validateBriefing(metadata datasetMeta, changes validationPublicationChanges, briefing validationBriefing) error { + if briefing.SchemaVersion != "1.0" || briefing.GeneratedAt != metadata.GeneratedAt || strings.TrimSpace(briefing.EditionID) == "" || strings.TrimSpace(briefing.Title) == "" || strings.TrimSpace(briefing.Scope) == "" { + return errorsForExtended("briefing provenance does not match metadata") + } + if briefing.Status != "ready" && briefing.Status != "unavailable" { + return fmt.Errorf("briefing has invalid status %q", briefing.Status) + } + if briefing.PublicationStatus != changes.Status || !briefing.ReviewRequired || len(briefing.Caveats) < 3 || !validBriefingHref(briefing.EvidenceEntryPoint) { + return errorsForExtended("briefing does not preserve publication status, review, caveat, or evidence requirements") + } + for _, caveat := range briefing.Caveats { + if strings.TrimSpace(caveat) == "" { + return errorsForExtended("briefing caveats must not be blank") + } + } + email := briefing.Email + if strings.TrimSpace(email.Subject) == "" || strings.TrimSpace(email.Preview) == "" || strings.TrimSpace(email.Markdown) == "" || strings.TrimSpace(email.CTALabel) == "" || !validBriefingHref(email.CTAPath) || email.SendPolicy != "manual_review_required" || strings.TrimSpace(email.PrimaryGoal) == "" { + return errorsForExtended("briefing email draft is incomplete or not review-gated") + } + carousel := briefing.SocialCarousel + if carousel.Format != "carousel_copy" || carousel.AspectRatio != "4:5" || carousel.ReviewPolicy != "manual_review_required" { + return errorsForExtended("briefing carousel contract is invalid") + } + if briefing.Status == "unavailable" { + if len(briefing.Signals) != 0 || len(carousel.Slides) != 0 { + return errorsForExtended("unavailable briefing must not publish signals or carousel slides") + } + return nil + } + if !monthPattern.MatchString(briefing.LatestPeriod) || !monthPattern.MatchString(briefing.PreviousPeriod) || briefing.LatestPeriod <= briefing.PreviousPeriod || !strings.Contains(email.Markdown, "{{BASE_URL}}") { + return errorsForExtended("ready briefing has invalid periods or unresolved delivery template") + } + wantKinds := []string{"reporter_total_change", "anchor_share_shift", "product_total_change"} + if len(briefing.Signals) != len(wantKinds) { + return fmt.Errorf("ready briefing has %d signals, want %d", len(briefing.Signals), len(wantKinds)) + } + seenIDs := make(map[string]struct{}, len(briefing.Signals)) + for index, signal := range briefing.Signals { + if signal.Kind != wantKinds[index] || strings.TrimSpace(signal.ID) == "" || strings.TrimSpace(signal.Title) == "" || strings.TrimSpace(signal.Summary) == "" || !iso3Pattern.MatchString(signal.ReporterISO3) || strings.TrimSpace(signal.ReporterName) == "" || !monthPattern.MatchString(signal.Period) || !monthPattern.MatchString(signal.PreviousPeriod) || signal.Period <= signal.PreviousPeriod || strings.TrimSpace(signal.Interpretation) == "" || strings.TrimSpace(signal.MeasurementLimit) == "" { + return fmt.Errorf("briefing has incomplete signal %+v", signal) + } + if _, exists := seenIDs[signal.ID]; exists { + return fmt.Errorf("briefing repeats signal id %q", signal.ID) + } + seenIDs[signal.ID] = struct{}{} + if signal.Kind == "product_total_change" { + if strings.TrimSpace(signal.Classification) == "" || !hs6Pattern.MatchString(signal.Code) || strings.TrimSpace(signal.Label) == "" { + return fmt.Errorf("briefing product signal is incomplete %+v", signal) + } + } else if signal.Classification != "" || signal.Code != "" || signal.Label != "" { + return fmt.Errorf("briefing aggregate signal claims product identity %+v", signal) + } + if err := validateBriefingObservedValue(signal.Current); err != nil { + return fmt.Errorf("briefing signal %s current value: %w", signal.ID, err) + } + if err := validateBriefingObservedValue(signal.Previous); err != nil { + return fmt.Errorf("briefing signal %s previous value: %w", signal.ID, err) + } + if !isFinite(signal.DeltaTradeUSD) || !approximatelyEqual(signal.DeltaTradeUSD, signal.Current.TotalTradeUSD-signal.Previous.TotalTradeUSD) || !isFinite(signal.ChinaShareDelta) || !approximatelyEqual(signal.ChinaShareDelta, signal.Current.ChinaShare-signal.Previous.ChinaShare) { + return fmt.Errorf("briefing signal %s has inconsistent deltas", signal.ID) + } + if signal.Previous.TotalTradeUSD > 0 { + if signal.ChangeRatio == nil || !isFinite(*signal.ChangeRatio) || !approximatelyEqual(*signal.ChangeRatio, signal.DeltaTradeUSD/signal.Previous.TotalTradeUSD) { + return fmt.Errorf("briefing signal %s has inconsistent change ratio", signal.ID) + } + } else if signal.ChangeRatio != nil { + return fmt.Errorf("briefing signal %s must omit a ratio with a zero baseline", signal.ID) + } + if len(signal.Evidence) < 2 { + return fmt.Errorf("briefing signal %s has insufficient evidence links", signal.ID) + } + for _, href := range signal.Evidence { + if !validBriefingHref(href) { + return fmt.Errorf("briefing signal %s has invalid evidence href %q", signal.ID, href) + } + } + } + if briefing.Signals[0].Period != briefing.LatestPeriod || briefing.Signals[0].PreviousPeriod != briefing.PreviousPeriod { + return errorsForExtended("briefing edition periods do not match the leading signal") + } + wantRoles := []string{"cover", "scale", "anchor_balance", "product", "method", "cta"} + if len(carousel.Slides) != len(wantRoles) { + return fmt.Errorf("briefing carousel has %d slides, want %d", len(carousel.Slides), len(wantRoles)) + } + for index, slide := range carousel.Slides { + if slide.Order != index+1 || slide.Role != wantRoles[index] || strings.TrimSpace(slide.Headline) == "" || strings.TrimSpace(slide.Body) == "" || len(slide.Evidence) == 0 { + return fmt.Errorf("briefing carousel has invalid slide %+v", slide) + } + for _, href := range slide.Evidence { + if !validBriefingHref(href) { + return fmt.Errorf("briefing carousel slide %d has invalid evidence href %q", slide.Order, href) + } + } + } + return nil +} + +func validateBriefingObservedValue(value validationBriefingObservedValue) error { + values := []float64{value.USATradeUSD, value.ChinaTradeUSD, value.TotalTradeUSD, value.ChinaShare} + for _, item := range values { + if !isFinite(item) || item < 0 { + return fmt.Errorf("contains invalid nonnegative value %v", item) + } + } + if value.ChinaShare > 1 || !approximatelyEqual(value.TotalTradeUSD, value.USATradeUSD+value.ChinaTradeUSD) || (value.TotalTradeUSD > 0 && !approximatelyEqual(value.ChinaShare, value.ChinaTradeUSD/value.TotalTradeUSD)) || (value.TotalTradeUSD == 0 && value.ChinaShare != 0) { + return errorsForExtended("totals or China share are inconsistent") + } + return nil +} + +func validBriefingHref(href string) bool { + return strings.HasPrefix(href, "./") && !strings.Contains(href, "..") && !strings.ContainsAny(href, "\r\n") +} + func sortedUnique(values []string) bool { if !sort.StringsAreSorted(values) { return false diff --git a/docs/DATA_SCHEMA.md b/docs/DATA_SCHEMA.md index 08e0587..eccf5a1 100644 --- a/docs/DATA_SCHEMA.md +++ b/docs/DATA_SCHEMA.md @@ -1,6 +1,6 @@ # Published data schema 2.0 -TradeGravity publishes one versioned artifact set under `site/data/`. Headline, product, strategic HS6, semiconductor, publish-to-publish change, tariff, bilateral-matrix, mirror-diagnostic, quality, and explanation artifacts share the trade publication timestamp. The semiconductor reference has its own additive schema and records both its editorial update date and publisher `generated_at`. `catalog.json` and `changes.json` have independent additive schemas and the same publisher timestamp. `context.json` has its own refresh time because it is built before the trade publisher. +TradeGravity publishes one versioned artifact set under `site/data/`. Headline, product, strategic HS6, semiconductor, publish-to-publish change, distribution briefing, tariff, bilateral-matrix, mirror-diagnostic, quality, and explanation artifacts share the trade publication timestamp. The semiconductor reference has its own additive schema and records both its editorial update date and publisher `generated_at`. `catalog.json`, `changes.json`, and `briefing.json` have independent additive schemas and the same publisher timestamp. `context.json` has its own refresh time because it is built before the trade publisher. ## Time and comparison semantics @@ -24,6 +24,7 @@ TradeGravity publishes one versioned artifact set under `site/data/`. Headline, | `semiconductors/monthly/index.json` | Focused monthly reporter/period discovery | UN Comtrade + semiconductor registry | | `semiconductors/monthly/{ISO3}.json` | Selected HS6 monthly USA/China flows | UN Comtrade | | `changes.json` | Previous-publication coverage, row, and value deltas for the focused monthly semiconductor layer | Publisher comparison of consecutive publications | +| `briefing.json` | Review-gated email and 4:5 carousel drafts derived from three cited monthly semiconductor signals | Deterministic publisher calculations over the focused monthly layer | | `tariffs/index.json` | Importer/year tariff partition discovery | WITS/TRAINS | | `tariffs/{ISO3}/{YEAR}.json` | Revision-aware strategic HS6 tariff rows | WITS/TRAINS | | `bilateral-matrix/index.json` | Multi-partner `TOTAL` partition discovery | UN Comtrade | @@ -39,7 +40,7 @@ TradeGravity publishes one versioned artifact set under `site/data/`. Headline, The catalog is the stable discovery layer for a dashboard that may grow beyond a few single-file datasets. Each resource declares an `id`, display title, `status`, analytical `grain`, `partitioning`, and an `href` only when an artifact is published. Current statuses are `ready`, `partial`, and `planned`. -`strategic_hs6`, `semiconductor_atlas`, `semiconductor_monthly`, `publication_changes`, `tariff_schedules`, `bilateral_matrix`, and `mirror_reconciliation` are published resources. The last ID is retained for catalog compatibility, but its title and artifact explicitly describe **unadjusted mirror-reporting diagnostics**, not a reconciled truth. Computed value-added networks and versioned scenario results remain planned contracts and do not claim that those observations exist. Published resources use relative same-origin paths; the validator rejects duplicate IDs, invalid statuses, unsafe paths, and metadata that conflicts with `meta.json`. +`strategic_hs6`, `semiconductor_atlas`, `semiconductor_monthly`, `publication_changes`, `distribution_briefing`, `tariff_schedules`, `bilateral_matrix`, and `mirror_reconciliation` are published resources. The last ID is retained for catalog compatibility, but its title and artifact explicitly describe **unadjusted mirror-reporting diagnostics**, not a reconciled truth. Computed value-added networks and versioned scenario results remain planned contracts and do not claim that those observations exist. Published resources use relative same-origin paths; the validator rejects duplicate IDs, invalid statuses, unsafe paths, and metadata that conflicts with `meta.json`. The current product resource demonstrates the intended scaling pattern: a small discovery index plus one reporter file. Higher-volume resources should use period, reporter, importer, industry, or sector chunks named in the catalog rather than expanding `latest.json`. @@ -209,6 +210,27 @@ Only mapped HS6 codes, monthly periods, and USA/CHN partners are admitted. The i `status` is `baseline` when no earlier comparable publication exists, `unchanged` when a previous publication exists but no admitted change is found, and `changed` otherwise. A baseline never masquerades as “no change.” The revision list is capped at 20 rows and ordered by the sum of absolute USA and China trade-value changes; global counts remain complete even when the list is truncated. Added and removed rows are counted but not represented as value revisions. The validator checks current-index identity, timestamps, dimensions, counts, finite values, revision arithmetic, and descending magnitude. +## Reviewed distribution briefing + +`briefing.json` uses schema `1.0` and converts the focused monthly semiconductor observations into one inspectable distribution contract. It contains exactly three deterministic signals when two comparable months exist: the largest absolute reporter total change, the largest absolute two-anchor China-share shift, and the largest absolute HS6 product change. Each signal retains current and previous USA/China values, arithmetic deltas, periods, interpretation limits, and relative evidence paths. + +The same signals feed two non-publishing draft formats: + +- `email` contains subject, preview, Markdown body, evidence CTA, `send_policy: "manual_review_required"`, and a primary goal; +- `social_carousel` contains six cited `4:5` copy slides with `review_policy: "manual_review_required"` in cover, scale, anchor-balance, product, method, and CTA order. + +The top-level `review_required` must remain `true`. A ready artifact is rejected if either channel permits automatic publication. If two comparable months are unavailable, the artifact fails closed with `status: "unavailable"`, no signals, and no sendable copy. The static application can download drafts and copy the evidence link; it does not maintain a subscriber database, send mail, or call a social publishing API. The offline distributor can render matched SVG and 1080×1350 PNG drafts from the validated slide model, but its manifest keeps both send and social authorization false. + +An optional local `approval.json` schema `1.0` is created only after the distributor verifies the exact manifest file set, byte counts, and SHA-256 digests. The manifest's carousel record includes the selected native `theme` (`intelligence-dark` or `editorial-light`) and fixed `caption_path: "carousel/caption.md"`. The caption is derived from the validated signals with period, evidence entry point, scope note, restrained tags, and a 1,800-rune project ceiling. A theme, caption, or image change alters hashes and invalidates an earlier approval. The approval records `scope: "content_release"`, the edition and manifest digest, reviewer, non-sensitive audience label, canonical UTC approval time, sorted channels, and fixed review attestations; automatic publishing remains false. + +An optional external `instagram-preflight.json` schema `1.0` binds an Instagram-approved manifest and approval digest to the theme, six-slide count, 1080×1350 dimensions, caption rune/hashtag counts, alt-text section count, and boolean integrity checks. It contains no caption text, hashtags, evidence URL, credentials, or account identity. `manual_upload_required` is true and `automatic_publish_authorized` is false. It must remain outside the approved kit; see [INSTAGRAM_PREFLIGHT.md](INSTAGRAM_PREFLIGHT.md). + +An optional local `delivery-preflight.json` schema `1.0` can then bind an email-approved edition to the SHA-256 digests of private subscriber and suppression CSVs. It records only aggregate consented, suppressed, suppression-row, and eligible counts; the approved audience label; the template, manifest, and approval digests; an explicit pilot ceiling; required unsubscribe headers and DKIM coverage; and whether individual opaque HTTPS unsubscribe URLs passed validation. It contains no recipient addresses, unsubscribe URLs, tokens, or local file paths. `provider_configured` and `delivery_authorized` remain false. The private CSV schemas and operational limits are documented in [EMAIL_DELIVERY_PREFLIGHT.md](EMAIL_DELIVERY_PREFLIGHT.md); neither the CSVs nor this local plan belongs under `site/data/`. + +The optional private subscription registry is deliberately outside the published schema. Its SQLite tables keep random subscription and pending-confirmation IDs, normalized email, audience, active/suppressed state, double-opt-in evidence, confirmation expiry/dispatch state, privacy-notice version, suppression reason/time, provider-feedback event IDs, and global bounced/complaint/provider-suppression state. Purpose-separated HMAC token payloads contain only a version and random ID. Confirmation GET is read-only and explicit POST records consent; globally suppressed addresses cannot receive confirmation or become active. Database files, WAL/SHM files, source/export CSVs, tokens, and secrets must never be published. See [UNSUBSCRIBE_SERVICE.md](UNSUBSCRIBE_SERVICE.md). + +An optional local `email-launch-authorization.json` schema `1.0` binds one Resend pilot to the exact preflight digest, edition, audience, sender, content/input digests, eligible count, operator, attestations, and a validity window of no more than one hour. It contains no recipient address or unsubscribe token. The accompanying private delivery-ledger SQLite schema stores only HMAC-derived recipient and delivery keys, aggregate labels, content/idempotency and authorization identifiers, provider message IDs, statuses, timestamps, and bounded non-sensitive reconciliation labels. Accepted and unresolved-pending rows both prevent an automatic resend. A provider-confirmed `not_accepted` resolution can transition back to a pending attempt only when a different launch-authorization digest is supplied later; the audit row and stable provider idempotency key are retained. Neither artifact is published, and neither changes the public `catalog.json`; see [EMAIL_PROVIDER_PILOT.md](EMAIL_PROVIDER_PILOT.md). + ## Multi-partner bilateral matrix The matrix index has `product_code: "TOTAL"`, `product_level: 0`, sorted reporter/partner/period dimensions, partition counts, partner-row counts, and source-observation counts. Each reporter/year file contains one row per alphabetic ISO3 partner: @@ -256,7 +278,7 @@ Run the same validation used before deployment: go run ./cmd/validator -dir site/data -min-reporters 40 ``` -The validator checks cross-file provenance and counts, reporter and period uniqueness, finite numbers, calculated totals/shares/balances, monthly product identities, mirror-pair arithmetic and disclosure, flow-availability identities, strategic registry membership, free/public reference policy, tariff rate identities, catalog contracts, context coverage, collection-run metadata, and every explanation citation. +The validator checks cross-file provenance and counts, reporter and period uniqueness, finite numbers, calculated totals/shares/balances, monthly product identities, briefing signal arithmetic and review gates, mirror-pair arithmetic and disclosure, flow-availability identities, strategic registry membership, free/public reference policy, tariff rate identities, catalog contracts, context coverage, collection-run metadata, and every explanation citation. ## CSV and filtered JSON diff --git a/docs/DISTRIBUTION.md b/docs/DISTRIBUTION.md new file mode 100644 index 0000000..380a713 --- /dev/null +++ b/docs/DISTRIBUTION.md @@ -0,0 +1,106 @@ +# Reviewed distribution workflow + +TradeGravity uses one evidence contract for its website briefing, email draft, and future social cards. The workflow is deliberately split into **analysis**, **rendering**, **editorial approval**, and **delivery** so that generating an asset never authorizes sending or publishing it. + +## Build a local kit + +Start from a validator-accepted `briefing.json` and the public URL at which its cited evidence will be available: + +```bash +go run ./cmd/distributor \ + -briefing examples/sample-data/briefing.json \ + -out distribution-kit \ + -base-url https://elecpapaya.github.io/TradeGravity/ \ + -theme intelligence-dark +``` + +`-theme` accepts `intelligence-dark` (the default analytical dark treatment) or `editorial-light` (a restrained light editorial treatment). Both are original, network-free native Go themes over the same validated six-slide model. Theme selection changes the PNG/SVG bytes and is recorded in the manifest, so changing it requires a new kit and approval. + +The command refuses unavailable or automatically publishable briefing contracts, insecure public base URLs, and an output directory that already exists. It performs no network request and sends nothing. + +The generated directory contains: + +```text +distribution-kit/ +├── manifest.json +├── REVIEW.md +├── email/ +│ ├── subject.txt +│ ├── preview.txt +│ ├── body.md +│ └── body.html +└── carousel/ + ├── index.html + ├── alt-text.md + ├── caption.md + ├── slide-01.svg … slide-06.svg + └── slide-01.png … slide-06.png +``` + +After review, the approval command adds `approval.json` beside the manifest. It is intentionally absent from a newly built kit. + +`carousel/caption.md` is a review-pending Instagram caption derived only from the three validated signals. It retains the comparison period, evidence entry point, scope warning, and restrained topic tags. The project imposes a 1,800-rune editorial ceiling. Edit the source briefing and regenerate instead of changing the caption independently. + +`manifest.json` records the edition, public evidence base, email CTA, selected native theme, caption path, 1080×1350 dimensions, available `png` and `svg` formats, review-pending state, explicit false send/publish authorization, and a byte count and SHA-256 digest for every reviewable file. The manifest is deterministic for the same briefing, base URL, and theme. + +## Editorial review + +Open `carousel/index.html` locally and preview `email/body.html` on desktop and mobile. Complete `REVIEW.md` before moving any file to an external provider. At minimum, verify: + +- all periods, values, directions, and source links against the cited JSON; +- month-to-month movement is not confused with a publish-to-publish revision; +- the email has one primary CTA back to the evidence; +- the six 4:5 cards remain legible at feed size and have reviewed alt text; and +- caveats do not become causal, routing, capacity, alignment, or investment claims. + +The SVG files are editable, resolution-independent originals. The matching PNG files are deterministic 1080×1350 raster assets generated from the same validated slide model; `carousel/index.html` deliberately previews those PNGs so the editor reviews the files intended for upload. Both formats and their SHA-256 hashes are created together, but they remain drafts. If copy or citations change, regenerate the whole kit instead of editing a PNG in place. Platform preview, alt-text entry, caption review, and the final Instagram publish action stay manual. + +The PNG renderer uses embedded Go fonts and makes no network request or external-font fetch. This keeps the output repeatable in CI and avoids silently changing typography between builds. Review every card at actual feed size because successful decoding and correct dimensions do not prove platform acceptance or reader comprehension. + +Both native palettes are tested across all three gradient stops. Normal text colors must retain at least 4.5:1 contrast, and the 20px bold role labels must retain at least 3:1 against their translucent pills. This automated contrast floor complements—rather than replaces—the feed-size visual and assistive-technology review. + +## Record content approval + +Complete `REVIEW.md`, then bind the unchanged manifest and all 20 tracked files to a named reviewer, non-sensitive audience label, channel list, and explicit UTC time: + +```bash +go run ./cmd/distribution-approval \ + -kit distribution-kit \ + -reviewer elecpapaya \ + -audience consented-internal-pilot \ + -channels email,instagram \ + -approved-at 2026-07-17T12:00:00Z \ + -attest-reviewed +``` + +The command fails if any tracked file was changed or removed, an untracked file was added, the manifest no longer has its review-pending/false-authorization gates, a channel is unsupported, or `approval.json` already exists. Rebuild the entire kit to make a new approval. Never place recipient addresses or provider secrets in the kit or audience label. + +`approval.json` has `scope: "content_release"` and binds the manifest's SHA-256 digest, file count, edition, reviewer, audience label, approved channels, time, and fixed attestations. It deliberately keeps `provider_delivery_ready`, `subscriber_consent_ready`, and `automatic_publish_ready` false. A future delivery adapter can call the same verifier and require the intended channel, but must enforce its own consent, suppression, sender, and provider gates. + +The approval record proves consistency with the local manifest; it does not authenticate the human reviewer by itself. Preserve it in a protected commit, release artifact, or separately signed record if reviewer authenticity is required. + +## Validate the manual Instagram package + +After an Instagram-channel approval, run `cmd/instagram-preflight` to recheck the unchanged manifest, six PNG dimensions, caption evidence/scope/tags, and all six alt-text sections. The aggregate output remains outside the kit and cannot publish anything. See [Instagram manual-publish preflight](INSTAGRAM_PREFLIGHT.md). + +## Manual GitHub Actions build + +The **Build reviewed distribution kit** workflow lets the operator choose either supported native theme, reads the validator-accepted `gh-pages` publication, builds the kit, verifies the theme and false social authorization in the manifest, and uploads it as a 14-day Actions artifact. It is manually dispatched and has read-only repository permissions. It does not have an email token, subscriber list, or social credential. + +## Delivery gate + +Provider-backed sending remains a separate operator-controlled step. Before enabling it, document and test: + +- double opt-in and the exact subscription promise; +- one-click unsubscribe and a durable suppression list; +- sender identity, SPF, DKIM, and DMARC; +- bounce and complaint handling; +- privacy notice, retention, deletion, and data-processing terms; +- secret storage outside the browser and generated artifact; and +- an approval record that binds one reviewed edition and its final hashes to one channel and audience. + +The content-release record implements only the editorial part of the last item. Run the provider-neutral [email delivery preflight](EMAIL_DELIVERY_PREFLIGHT.md) locally to validate a private double-opt-in CSV, apply a private suppression CSV, and create an aggregate-only plan. The preflight keeps provider configuration and delivery authorization false. + +The [provider-backed email pilot](EMAIL_PROVIDER_PILOT.md) adds a second, short-lived launch approval plus a fail-closed Resend adapter. It replays the exact preflight inputs immediately before sending and records accepted or uncertain attempts in a private HMAC-keyed SQLite ledger. Merely building, approving, or preflighting a kit still cannot send email; the live command additionally requires the matching authorization, provider and ledger secrets, and `-send-live`. + +TradeGravity should not use open tracking pixels by default. If aggregate link measurement is later added, disclose it and keep raw subscriber behavior out of the public dataset. diff --git a/docs/EMAIL_DELIVERY_PREFLIGHT.md b/docs/EMAIL_DELIVERY_PREFLIGHT.md new file mode 100644 index 0000000..cee97bd --- /dev/null +++ b/docs/EMAIL_DELIVERY_PREFLIGHT.md @@ -0,0 +1,79 @@ +# Email delivery preflight + +TradeGravity keeps subscriber addresses outside the repository, static site, distribution kit, approval record, and aggregate preflight artifact. The preflight CLI is a provider-neutral gate between reviewed content and any future network sender. It validates double opt-in evidence, applies a durable suppression list, enforces a small pilot ceiling, and then emits only counts and source-file digests. It sends no email. + +## Inputs + +Start with a distribution kit whose `approval.json` includes the `email` channel. Keep both CSV files in a private local directory outside the kit. + +The subscriber CSV header must be exactly: + +```csv +email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url +reader@example.invalid,consented-internal-pilot,active,2026-07-10T01:00:00Z,double_opt_in,website-form,v1,https://subscriptions.example.invalid/u/opaque-recipient-token +``` + +Every subscriber row must: + +- match the non-sensitive audience label in `approval.json`; +- have `status` equal to `active` and `consent_method` equal to `double_opt_in`; +- contain an RFC 3339 consent time that is not in the future; +- record a consent source and privacy-notice version; and +- use a unique plain email addr-spec without a display name; and +- provide a unique absolute HTTPS unsubscribe URL containing an opaque recipient token, with no credentials, fragment, or decoded email address. + +The suppression CSV header must be exactly: + +```csv +email,reason,suppressed_at +former-reader@example.invalid,unsubscribed,2026-07-12T03:00:00Z +``` + +Allowed reasons are `unsubscribed`, `bounced`, `complaint`, `invalid`, and `manual`. An empty suppression list still needs the header. Suppression always wins over active consent. + +## Run the preflight + +```bash +go run ./cmd/distribution-preflight \ + -kit distribution-kit \ + -subscribers private/subscribers.csv \ + -suppressions private/suppressions.csv \ + -out delivery-preflight.json \ + -generated-at 2026-07-17T12:30:00Z \ + -max-recipients 25 +``` + +The command refuses: + +- a kit without a valid email content approval; +- changed, missing, or untracked kit files; +- subscriber or suppression inputs stored inside the kit; +- malformed, duplicate, future-dated, wrong-audience, or non-double-opt-in subscriber rows; +- malformed or duplicate suppression rows; +- an empty post-suppression audience; and +- an eligible audience above the explicit pilot limit. + +`delivery-preflight.json` uses file mode `0600` where the operating system supports it and refuses to overwrite an existing file. It records the edition, manifest and approval digests, audience label, subscriber/suppression file digests, aggregate counts, template digest, required unsubscribe headers, and the pilot limit. It contains no recipient addresses, unsubscribe URLs, tokens, or local source paths. + +## What remains intentionally false + +A passing plan has: + +```json +{ + "consent_validated": true, + "suppression_applied": true, + "unsubscribe_urls_validated": true, + "contains_recipient_addresses": false, + "provider_configured": false, + "delivery_authorized": false +} +``` + +The approved HTML and Markdown templates contain exactly one `{{UNSUBSCRIBE_URL}}` placeholder in addition to the single primary evidence CTA. A future sender must replace that placeholder with a recipient-specific HTTPS unsubscribe URL, set `List-Unsubscribe` to that HTTPS URI, set `List-Unsubscribe-Post` to `List-Unsubscribe=One-Click`, and ensure a valid DKIM signature covers both headers. These are the one-click sender requirements defined by [RFC 8058](https://www.rfc-editor.org/rfc/rfc8058.html); Gmail's current sender guidance also requires a visible body link for subscribed messages and one-click support for high-volume senders. ([Gmail sender guidelines](https://support.google.com/mail/answer/81126)) + +The reference [provider-backed email pilot](EMAIL_PROVIDER_PILOT.md) re-runs the same in-memory preflight immediately before sending, requires unchanged source digests, and obtains a separate short-lived launch authorization. The operator must still authenticate the sender domain and deploy/test the signed Resend feedback handler so bounce, complaint, and provider-suppression events update the private registry before issuing that authorization. + +For a provider-neutral implementation that generates these URLs and exports both private CSVs, see the [subscription registry and unsubscribe service](UNSUBSCRIBE_SERVICE.md). It imports existing double-opt-in evidence; it does not create or prove the original opt-in. + +Do not upload the CSV inputs to GitHub Actions artifacts. Do not print the eligible in-memory address list, embed it in logs, or copy it into the aggregate JSON. A passing preflight is never a live-send authorization. diff --git a/docs/EMAIL_PROVIDER_PILOT.md b/docs/EMAIL_PROVIDER_PILOT.md new file mode 100644 index 0000000..f90345f --- /dev/null +++ b/docs/EMAIL_PROVIDER_PILOT.md @@ -0,0 +1,121 @@ +# Provider-backed email pilot + +TradeGravity includes a bounded Resend adapter for a deliberately small, consented email pilot. It is not connected to the static site or GitHub Actions and it does not run on a schedule. A live provider request is possible only when the reviewed kit, private subscriber and suppression inputs, aggregate preflight, short-lived launch authorization, provider API key, delivery-ledger secret, and explicit `-send-live` acknowledgement all agree. + +The adapter follows one-email/one-job lifecycle guidance: each semiconductor brief has one primary evidence CTA, one visible unsubscribe link, and no tracking pixel. It sends one provider request per recipient rather than placing multiple readers in `To`, `Cc`, or `Bcc`. + +Audience consent may be imported from an existing verified double-opt-in source or collected by the default-off signup/confirmation flow documented in [UNSUBSCRIBE_SERVICE.md](UNSUBSCRIBE_SERVICE.md). Confirmation mail is transactional and does not itself add an address to a briefing audience; only the confirmation-page POST does so. + +## Security boundary + +Keep these files and values outside the repository, distribution kit, public site, CI logs, and Actions artifacts: + +- subscriber and suppression CSVs; +- `delivery-preflight.json` and `email-launch-authorization.json`; +- `delivery-ledger.db` plus its WAL/SHM files and backups; +- `RESEND_API_KEY`; and +- `TRADEGRAVITY_DELIVERY_SECRET`, a stable random value of at least 32 bytes. + +The delivery ledger stores edition and audience labels, content digests, provider/idempotency identifiers, times, and HMAC-derived recipient keys. It does not store recipient addresses, unsubscribe URLs, or rendered bodies. File mode is requested as `0600` where supported, but the operator still needs encrypted storage, access control, backups, retention, and deletion procedures. + +The Resend endpoint is compiled as `https://api.resend.com/emails`; it cannot be redirected with a CLI flag that might exfiltrate the API key. HTTP timeouts and response-size limits are enforced, and provider error bodies are not printed. + +## 1. Produce the aggregate preflight + +Follow [Email delivery preflight](EMAIL_DELIVERY_PREFLIGHT.md) and keep the resulting files private: + +```bash +go run ./cmd/distribution-preflight \ + -kit distribution-kit \ + -subscribers private/subscribers.csv \ + -suppressions private/suppressions.csv \ + -out private/delivery-preflight.json \ + -generated-at 2026-07-17T12:30:00Z \ + -max-recipients 25 +``` + +## 2. Record a short-lived launch approval + +Verify the Resend sender domain, SPF, DKIM, and DMARC; deploy and test both the HTTPS unsubscribe endpoint and the signed feedback endpoint described in [UNSUBSCRIBE_SERVICE.md](UNSUBSCRIBE_SERVICE.md); prove that `email.bounced`, `email.complained`, and `email.suppressed` events become durable suppressions; review privacy controls; and inspect the final eligible pilot list. Then create an authorization that expires within one hour: + +```bash +go run ./cmd/email-launch-approval \ + -preflight private/delivery-preflight.json \ + -out private/email-launch-authorization.json \ + -provider resend \ + -from 'TradeGravity ' \ + -reply-to maintainer@example.org \ + -authorized-by elecpapaya \ + -authorized-at 2026-07-17T12:35:00Z \ + -expires-at 2026-07-17T13:05:00Z \ + -attest-domain-authenticated \ + -attest-feedback-ready \ + -attest-privacy-reviewed \ + -attest-pilot-recipients +``` + +The JSON contains aggregate identities and digests but no recipient address or token. It refuses overwrite. Changing the preflight, manifest, approval, email template, subscriber file, suppression file, recipient count, audience, provider, or sender requires a new authorization. + +## 3. Send the bounded pilot + +Set secrets without writing them into shell history, then run within the authorization window: + +```bash +export RESEND_API_KEY='read-from-your-secret-manager' +export TRADEGRAVITY_DELIVERY_SECRET='at-least-32-stable-random-bytes' + +go run ./cmd/email-delivery \ + -kit distribution-kit \ + -subscribers private/subscribers.csv \ + -suppressions private/suppressions.csv \ + -preflight private/delivery-preflight.json \ + -authorization private/email-launch-authorization.json \ + -ledger private/delivery-ledger.db \ + -send-at 2026-07-17T12:40:00Z \ + -send-live +``` + +Immediately before the first provider request, the command reruns content approval, double-opt-in, suppression, audience, URL, and pilot-ceiling checks. It compares all live digests and counts with both the original preflight and launch approval. Each request includes: + +- one recipient; +- the approved HTML and Markdown bodies with that recipient's HTTPS link; +- `List-Unsubscribe: `; +- `List-Unsubscribe-Post: List-Unsubscribe=One-Click`; and +- a stable provider `Idempotency-Key` derived from the edition and a secret HMAC recipient identity. + +Resend documents custom email headers and `Idempotency-Key` on `POST /emails`; its idempotency window is currently 24 hours. The local ledger is therefore the longer-lived duplicate guard. See [Resend Send Email](https://resend.com/docs/api-reference/emails/send-email) and [Resend idempotency keys](https://resend.com/docs/dashboard/emails/idempotency-keys). + +## Failure and reconciliation + +An accepted ledger row is skipped on every later run. If the process cannot prove provider acceptance, it leaves a `pending` row and stops. A later run refuses to resend that recipient automatically—even if Resend's 24-hour idempotency window has elapsed. This trades possible non-delivery for protection against an accidental duplicate. + +Do not delete or edit the ledger to force a retry. First reconcile the pending idempotency key against the provider dashboard and logs. The signed feedback handler records later suppression outcomes, but it cannot prove whether an unresolved send request was accepted; that delivery-ledger reconciliation remains an operator decision. Treat any pending row as a stopped pilot until the provider outcome is known. + +If the provider confirms acceptance, record the exact provider message ID. The recipient argument is used only in memory to derive the existing HMAC ledger key and is neither printed nor stored: + +```bash +go run ./cmd/email-delivery-reconcile \ + -ledger private/delivery-ledger.db \ + -edition EDITION_ID \ + -audience consented-internal-pilot \ + -recipient 'private-recipient@example.org' \ + -outcome accepted \ + -provider-message-id PROVIDER_MESSAGE_ID \ + -resolved-by elecpapaya \ + -evidence resend-dashboard-message-match \ + -resolved-at 2026-07-17T12:50:00Z \ + -attest-provider-checked +``` + +If the provider positively confirms that it did **not** accept the request, use `-outcome not_accepted` and omit `-provider-message-id`. That outcome remains in the ledger as an audit record. The same launch authorization still cannot retry it: create a new unexpired authorization file and run delivery at a later time. The provider idempotency key remains stable across the reconciled retry. + +`resolved-by`, `evidence`, and provider message IDs are bounded non-sensitive labels; addresses and URLs are rejected in the stored audit fields. A contradictory second outcome is rejected, while an identical reconciliation is idempotent. + +## Current limits + +- No live TradeGravity send has been performed or implied by these tools. +- Provider account creation, domain verification, billing, data-processing terms, webhook registration, and HTTPS deployment are external operator actions. +- Signed bounce, complaint, and provider-suppression ingestion is implemented, but public HTTPS deployment, provider webhook registration, secret rotation, monitoring, backup/restore, and a real replay drill remain operator responsibilities. +- Link/open tracking is not enabled; provider-account defaults must also be reviewed. +- The first pilot should remain small, manually observed, and limited to readers who explicitly expect this edition. +- Instagram cards remain review-gated manual uploads; email delivery never authorizes social publishing. diff --git a/docs/INSTAGRAM_PREFLIGHT.md b/docs/INSTAGRAM_PREFLIGHT.md new file mode 100644 index 0000000..5b1049b --- /dev/null +++ b/docs/INSTAGRAM_PREFLIGHT.md @@ -0,0 +1,41 @@ +# Instagram manual-publish preflight + +TradeGravity keeps Instagram publishing manual. `cmd/instagram-preflight` verifies that one unchanged, Instagram-approved distribution kit is ready for a private platform preview without configuring an account credential or authorizing a post. + +## Run after content approval + +First build and approve the kit for the `instagram` channel as described in [DISTRIBUTION.md](DISTRIBUTION.md). Then write the aggregate preflight outside the kit: + +```bash +go run ./cmd/instagram-preflight \ + -kit distribution-kit \ + -out instagram-preflight.json \ + -generated-at 2026-07-17T14:00:00Z +``` + +The command fails unless: + +- `approval.json` covers the `instagram` channel and matches the exact manifest; +- all six manifest-bound PNGs decode at 1080×1350; +- `caption.md` remains within TradeGravity's 1,800-rune editorial ceiling, contains the public evidence base and scope warning, has 1–8 unique simple hashtags, and has no unresolved placeholder; +- `alt-text.md` has one evidence section for each slide; and +- every kit file still matches its recorded byte count and SHA-256 digest. + +The output records only edition/theme identities, hashes, counts, dimensions, and boolean checks. It does not contain the caption, hashtags, evidence URL, credentials, or platform account. Its fixed controls remain: + +```json +{ + "contains_caption_text": false, + "contains_credentials": false, + "manual_upload_required": true, + "automatic_publish_authorized": false +} +``` + +The file is not part of the approved kit and the command refuses to write it inside that directory or overwrite an existing output. It is ignored at the repository root by default. + +## Private platform preview + +After preflight, use Instagram's own draft or private preview surface manually. Confirm ordering, crop, line wrapping, caption appearance, hashtag wording, alt-text entry, account identity, and rights. Do not treat a passing local preflight as proof that Instagram accepted or published the carousel. Rebuild and reapprove after any content change. + +Automatic publishing would require a separate credential-bearing adapter, explicit account and permission checks, platform idempotency, post-result reconciliation, and a new short-lived launch authorization. None of those capabilities are implemented or implied here. diff --git a/docs/ITERATION_LOG.md b/docs/ITERATION_LOG.md index c7615e9..e93d50b 100644 --- a/docs/ITERATION_LOG.md +++ b/docs/ITERATION_LOG.md @@ -158,3 +158,115 @@ No integration code or market data was added. The investigation produced [`docs/ ### Decision and next step Reject embedding Vibe-Trading, importing its internal Python modules, or using it as the current public site's live backend. Keep a read-only MCP/local companion as a conditional experiment. The next bounded step, if pursued, is Stage 0 only: select a legally usable provider and create a cited 8–12 issuer registry across at least four semiconductor stages before installing or invoking Vibe-Trading. + +## TG-2026-003 — Establish a deterministic notebook before a finance MCP + +- Date opened: 2026-07-17 +- Status: Proposed; implementation and provider testing have not started +- Scope: Semiconductor market-research handoff, reproducibility, and data rights +- Owner: TradeGravity maintainer + +### Observation + +TradeGravity already publishes versioned JSON and includes a small Jupyter example, but the proposed semiconductor-to-market handoff does not yet have a deterministic reference consumer. The request to compare Notebook and finance MCP options is a maintainer hypothesis, not evidence that users need a stock-research workflow. + +The [finance handoff assessment](research/FINANCE_HANDOFF_OPTIONS.md) found that notebook and MCP frameworks can both consume financial data, but neither supplies global listing coverage, correct corporate-action treatment, or redistribution rights by itself. A provider-backed MCP also adds an abstraction layer before TradeGravity has a reference result against which to test it. + +### Hypothesis and decision signal + +If a pinned Quarto or Jupyter analysis consumes a versioned TradeGravity handoff, a cited issuer registry, and synthetic or locally licensed market inputs, the project can test the value of the combined analysis while keeping calculations, provenance, missing states, and license boundaries inspectable. + +Advance to an MCP comparison only when all of the following are true: + +- the notebook executes from a clean environment without a browser secret or unrecorded network fallback; +- each result preserves provider, symbol, exchange/MIC, currency, listing type, price-adjustment basis, retrieval time, and unresolved status; +- split, dividend, ticker-change, stale-response, missing-day, currency, and ADR cases have deterministic tests; and +- a selected provider confirms the intended storage, display, derived-data, caching, and retention rights in writing. + +Stop or redesign if the leading provider cannot return primary listings and corporate actions across the bounded connector-market sample, or if an MCP result differs from the pinned direct-API reference or omits provenance. + +### Change + +No notebook, MCP server, provider account, API key, or market-data artifact was added. The investigation produced [`docs/research/FINANCE_HANDOFF_OPTIONS.md`](research/FINANCE_HANDOFF_OPTIONS.md), ranked the local and browser notebook surfaces, compared official finance data/MCP options, defined a canonical evidence contract, and specified a staged provider bake-off. + +### Evidence + +| Evidence type | Date | Result | Interpretation | +| --- | --- | --- | --- | +| Repository review | 2026-07-17 | An existing schema-2.0 Jupyter example reads TradeGravity's public JSON, while CI currently does not execute notebooks | Supports extending a familiar project pattern; does not establish that a market handoff is useful or reproducible yet | +| Official notebook-tool review | 2026-07-17 | Quarto, marimo, Jupyter/JupyterLite, and Observable can produce different combinations of static, local, and browser analysis; browser-WASM variants have package and secret constraints | Supports a pinned local report first; does not prove compatibility with the final package set | +| Official provider and MCP review | 2026-07-17 | Candidate services expose price, identifier, filing, and corporate-action capabilities, but plan coverage and redistribution conditions vary; broker-linked MCPs add out-of-scope write operations | Supports a direct, read-only provider bake-off before adopting MCP; does not select a canonical provider | +| Local deterministic proof | Not measured yet | No handoff schema, issuer registry, fixture, or report has been implemented | Required in Stage 0 | +| External-user value | Not measured yet | No participant has evaluated the combined report | Required before a public feature or MCP integration | + +### Decision and next step + +Prefer a Quarto `.qmd` plus a small tested Python calculation module as the first reference consumer; a plain Jupyter notebook remains the minimum-friction fallback. Defer finance MCP adoption and reject broker-capable MCPs for this scope. The next bounded step is Stage 0 only: define the handoff, issuer-registry, and market-evidence schemas, then render a no-network report from synthetic corporate-action fixtures and one cited TradeGravity snapshot. + +## TG-2026-004 — Derive reviewed distribution drafts from one evidence contract + +- Date opened: 2026-07-17 +- Status: Implemented; local verification complete; production and external evaluation pending +- Scope: Distribution, editorial safety, privacy, and adoption evidence +- Owner: TradeGravity maintainer + +### Observation + +The project can generate evidence-grounded analysis, but publishing a newsletter or Instagram carousel separately would risk inconsistent numbers, missing citations, and unsupported automation. TradeGravity is currently a static site with no consent store, sender identity, suppression list, or social-account publishing boundary. The request for email and card-news distribution is a maintainer product hypothesis, not evidence of audience demand. + +### Hypothesis and decision signal + +If the publisher derives email and social drafts from the same validated semiconductor observations and requires a human review before either leaves the repository, the maintainer can test distribution usefulness without introducing subscriber PII, browser-held secrets, or uncited claims. + +Retain the contract when all of the following are true: + +- every ready draft has the same three signal kinds, periods, arithmetic, and cited evidence as the published monthly artifacts; +- the deployed UI never claims that delivery is configured and never collects an email address; +- an editor can trace every email section and carousel slide to the cited artifact; and +- at least two of three intended readers say the brief gives them a reason to inspect the underlying Chip Lens. + +Do not enable provider sending or direct social publishing until double opt-in, unsubscribe/suppression handling, privacy retention, sender authentication, platform rights, and an explicit approval gate are documented and tested. + +### Change + +The implementation adds: + +- a deterministic `briefing.json` publisher contract with reporter-scale, two-anchor share-shift, and HS6 product signals; +- review-gated email Markdown and a cited six-slide 4:5 carousel-copy bundle; +- validator checks for provenance, periods, totals, shares, deltas, ratios, evidence paths, slide roles, and mandatory human review; +- Chip Lens controls for inspecting signals, downloading both drafts, and copying the evidence entry point; and +- a fail-closed unavailable state when two comparable monthly periods do not exist; and +- an offline review-kit builder that emits one-primary-CTA email HTML/Markdown, a cited Instagram caption, two native themes, six matched 1080×1350 SVG/PNG assets, alt text, an approval checklist, and a deterministic hash manifest, plus a read-only manual Actions workflow. +- a content-release approval contract and CLI that reject modified, missing, or untracked kit files before binding the manifest digest to a reviewer, audience label, time, and approved channels while leaving provider, consent, and automation readiness false. +- a local email preflight that verifies email content approval, exact private double-opt-in and suppression schemas, audience match, timestamps, unique opaque HTTPS unsubscribe URLs, suppression precedence, and a pilot ceiling before emitting an address/token-free aggregate plan with provider and delivery authorization false. +- a separate private SQLite subscription service that imports existing consent or runs a default-off double-opt-in form, sends short-lived confirmation mail with stable idempotency, activates only on explicit POST, exports HMAC links and suppression rows, verifies signed Resend feedback, and prevents globally suppressed addresses from reactivation. +- a fail-closed Resend pilot adapter that requires a one-hour launch approval bound to the exact preflight, sender, audience, content and input digests; reruns all preflight checks at send time; sends one recipient per request with visible/header one-click links; stores only HMAC recipient identities in a private duplicate-prevention ledger; and requires provider-evidence reconciliation plus a different launch approval before retrying confirmed non-acceptance. +- an aggregate Instagram manual-publish preflight that rechecks the approved kit, PNG dimensions, caption evidence/scope/tags, and alt-text completeness without storing content or credentials and without authorizing a post. + +No live provider account, API key, real subscriber record, tracking pixel, social credential, or automatic publishing action was added. The Resend adapter is explicit, local, and inert without private inputs, a matching short-lived authorization, two environment secrets, and `-send-live`. + +### Evidence + +| Evidence type | Date | Result | Interpretation | +| --- | --- | --- | --- | +| Go unit and static checks | 2026-07-17 | `go test ./...` and `go vet ./...` passed | Establishes deterministic calculations and validator behavior; does not establish production-provider coverage or editorial quality | +| Frontend contract and structure checks | 2026-07-17 | 45 Node tests and JavaScript syntax checks passed | Establishes browser helper behavior and expected UI wiring; does not establish real-user comprehension | +| Sample publication validation | 2026-07-17 | `cmd/validator` accepted the synthetic three-reporter publication with `briefing.json` | Establishes a network-free end-to-end artifact contract; synthetic values are not real market evidence | +| Distribution-kit build | 2026-07-17 | The sample briefing produced review-pending email plus six matched SVG/PNG cards with deterministic SHA-256 entries and explicit false send/publish authorization | Establishes repeatable production inputs and safety gates; does not establish provider delivery or audience value | +| Distribution-kit render | 2026-07-17 | Six SVGs and six embedded-font PNGs decoded at 1080×1350 with no external assets; representative scale, product, and method PNGs were visually inspected; email HTML rendered on desktop and 390px mobile with one primary CTA plus a utility unsubscribe link, no scripts or images, no horizontal overflow, and no console issues | Establishes bounded visual, dimension, and dependency checks; does not establish inbox-client compatibility, accessibility-tool results, Instagram acceptance, or reader comprehension | +| Content-approval gate | 2026-07-17 | A fixed-time sample approval verified the 20-file manifest, sorted email/Instagram channels, recorded three attestations, refused overwrite, and retained false provider/consent/automatic-publish readiness; tests rejected tampered caption/email content, missing, and untracked files plus manifest changes after approval | Establishes a reproducible content-to-manifest binding; it does not authenticate the reviewer, prove subscriber consent, configure a provider, or authorize a live send/post | +| Email consent preflight | 2026-07-17 | Reserved-domain fixtures verified two double-opt-in rows, unique opaque HTTPS unsubscribe URLs, one matched suppression, one eligible recipient, exact audience binding, address/token-free JSON, output non-overwrite, and a 25-recipient ceiling; negative tests rejected wrong audience, non-double-opt-in, future consent, full suppression, over-limit audiences, insecure/duplicate/address-exposing URLs, and inputs inside the kit | Establishes deterministic local consent/suppression and unsubscribe-input gating without persisting addresses or tokens in artifacts; it does not prove a production consent store, working unsubscribe endpoint, provider event handling, sender authentication, or live-delivery authorization | +| Subscription registry and endpoints | 2026-07-17 | Two reserved-domain consents imported into a mode-0600 SQLite registry; exported tokens decoded without email/audience data; GET retained both active rows; invalid-media POST was rejected; valid and repeated one-click POST produced one stable suppression; signed raw-body bounce feedback suppressed one active address, its replay was idempotent, a forged signature was rejected, and a provider-suppressed address was blocked from later import; registry exports passed the email preflight with no address/token leakage | Establishes the reference storage, token, unsubscribe, signed-feedback, replay, global-suppression, and export behavior locally; it does not prove TLS proxy configuration, production durability, original double opt-in, abuse resistance, privacy operations, provider registration, or public availability | +| Double-opt-in signup | 2026-07-17 | Reserved-domain tests created a short-lived pending request, verified identity-free purpose-separated tokens, kept confirmation GET read-only, activated on explicit POST, made repeats idempotent, retried uncertain confirmation delivery with the same provider key after cooldown, omitted unsubscribe headers before consent, rejected expired tokens, and withheld mail from a globally bounced address | Establishes local consent-state and confirmation-delivery behavior; it does not prove sender-domain authentication, inbox placement, public rate limiting, privacy operations, or production HTTPS durability | +| Native carousel theme seam | 2026-07-17 | Added an original `editorial-light` renderer alongside the default `intelligence-dark`; both produce deterministic matched SVG/PNG files from the same validated model, retain evidence and false publish authorization, bind the selected theme into the manifest, reject unknown themes, and pass the existing Instagram approval verifier. Representative cover, scale, and CTA PNGs were visually inspected at 1080×1350 with no clipping | Improves presentation choice without adopting an unescaped browser renderer or weakening provenance; it does not establish audience preference, Instagram acceptance, or automatic publishing rights | +| Carousel palette contrast | 2026-07-17 | Relative-luminance tests measured header, counter/evidence, headline, body, evidence title, and footer against every gradient stop at a minimum 4.5:1, plus 20px bold role labels against their 16%-accent pills at a minimum 3:1. The light muted color improved from a measured 4.07:1 worst case to above the floor, and the regenerated scale card was visually inspected | Establishes deterministic palette-level contrast; it does not replace zoom, low-vision, screen-reader, feed-size, or platform-client testing | +| Instagram caption draft | 2026-07-17 | The distributor generated a period-labelled caption from all three validated signals, included the evidence entry point and non-causal/non-investment scope note, enforced a project length ceiling, recorded its path and digest in the manifest, and rejected an independently replaced caption at approval | Closes the manual upload-content gap while preserving one-source consistency; it does not prove hashtag performance, link clickability in a client, platform acceptance, or editorial approval | +| Instagram manual preflight | 2026-07-17 | An aggregate-only CLI required an unchanged Instagram approval, decoded six 1080×1350 PNGs, checked caption length/evidence/scope/unique restrained tags and six alt-text evidence sections, rejected email-only approval and tampering, refused output inside the kit/overwrite, and retained false credential/content/publish flags | Establishes a reproducible local handoff to private platform preview; it does not prove account permissions, Instagram rendering, upload success, reach, or audience value | +| Provider pilot adapter | 2026-07-17 | Reserved-domain integration tests bound a short-lived launch approval to an unchanged preflight, sent two isolated requests through a synthetic provider, verified visible and RFC one-click links plus distinct idempotency keys, skipped accepted rows, and stopped an uncertain outcome. Provider-confirmed acceptance then remained skipped; confirmed non-acceptance rejected the same authorization and completed only under a different short-lived authorization while retaining its idempotency key. Repeated/contradictory resolutions and PII-bearing audit labels were tested, and a TLS mock verified the Resend HTTP payload and bounded errors | Establishes local launch, rendering, request, reconciliation, and duplicate-safety behavior without leaking addresses/tokens into authorization or ledger files; it does not prove a verified sender domain, inbox delivery, provider-account settings, public webhook operation, or a live consented audience | +| Maintainer browser inspection | 2026-07-17 | Synthetic publication rendered at 1440×1000 and 390×844 with three ready signals, enabled draft actions, no horizontal overflow, and no console warnings or errors | Establishes a bounded local responsive rendering check; does not replace assistive-technology or external-reader evaluation | +| Production artifact | Not measured yet | Public `data/briefing.json` has not been verified | Required after merge and a semiconductor refresh | +| External-reader value | Not measured yet | No reader has evaluated the email or carousel draft | Required before conducting the provider-backed pilot or automating any social distribution | + +### Decision and next step + +Keep the provider-neutral draft contract and deterministic PNG renderer as a release candidate. Next, verify the public artifact after deployment, use the exported Markdown with a small consented test audience, and privately preview the generated PNG sequence with reviewed alt text before selecting an email provider or enabling any social integration. Direct publishing remains a separate decision after citation legibility, rights, platform, and audience-value review. diff --git a/docs/UNSUBSCRIBE_SERVICE.md b/docs/UNSUBSCRIBE_SERVICE.md new file mode 100644 index 0000000..da2d112 --- /dev/null +++ b/docs/UNSUBSCRIBE_SERVICE.md @@ -0,0 +1,136 @@ +# Private subscription registry and unsubscribe service + +TradeGravity includes a private reference service for double-opt-in signup, working unsubscribe URLs, and durable suppressions. The static dashboard still never receives an address or API key. The separately deployed service can send a short transactional confirmation through Resend, activate consent only after an explicit confirmation POST, import existing verified consent, export private delivery CSVs, and verify signed provider feedback. + +## Security boundary + +Keep all of the following outside the repository, `site/data/`, GitHub Actions, Actions artifacts, and the distribution kit: + +- `subscriptions.db`, its WAL/SHM files, and backups; +- the source consent CSV; +- exported subscriber and suppression CSVs; and +- `TRADEGRAVITY_UNSUBSCRIBE_SECRET`; and +- `RESEND_WEBHOOK_SECRET` when provider feedback is enabled. +- `RESEND_API_KEY` when public signup is enabled. + +Use a dedicated directory on an encrypted volume with access limited to the operator account. The tools request mode `0600` for the database and exports where supported, but file mode alone is not encryption or a backup policy. + +The secret must contain at least 32 bytes and remain stable while issued links are in circulation. Store it in a secret manager or password manager. Rotating or losing it invalidates existing links. The token contains only a version and random 128-bit subscription ID, authenticated with HMAC-SHA-256; it contains no email address or audience label. + +## Import verified consent and create private exports + +The source CSV must be produced by a real double-opt-in process and have this exact header: + +```csv +email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version +reader@example.invalid,consented-internal-pilot,active,2026-07-10T01:00:00Z,double_opt_in,website-form,v1 +``` + +Set the same secret for registry and service processes, then import and export: + +```bash +export TRADEGRAVITY_UNSUBSCRIBE_SECRET='at-least-32-random-secret-bytes' + +go run ./cmd/subscription-registry \ + -db private/subscriptions.db \ + -base-url https://subscriptions.example.org/tradegravity/ \ + -consents private/verified-consents.csv \ + -audience consented-internal-pilot \ + -out-subscribers private/delivery-subscribers.csv \ + -out-suppressions private/delivery-suppressions.csv \ + -imported-at 2026-07-17T12:00:00Z +``` + +The command validates active double opt-in, exact audience labels, timestamps, uniqueness, source, and privacy-notice version. It refuses to overwrite exports. Reimporting an active record updates its consent evidence only when it is not older than the stored record. Reimporting a suppressed record never reactivates it. + +The subscriber export has the exact preflight schema and a stable, unique HTTPS unsubscribe URL for each active row. The suppression export contains unsubscribed addresses and timestamps. Feed both directly to [`cmd/distribution-preflight`](EMAIL_DELIVERY_PREFLIGHT.md); never copy them into the repository. + +## Enable double-opt-in signup + +Signup is disabled unless explicitly configured. After publishing the privacy notice and verifying the sender domain, enable it on the same private service: + +```bash +export TRADEGRAVITY_UNSUBSCRIBE_SECRET='the-same-stable-secret' +export RESEND_API_KEY='read-from-your-secret-manager' + +go run ./cmd/unsubscribe-service \ + -db private/subscriptions.db \ + -base-url https://subscriptions.example.org/tradegravity/ \ + -listen 127.0.0.1:8081 \ + -enable-signup \ + -signup-audience tradegravity-briefing \ + -consent-source public-subscribe-form \ + -privacy-notice-version v1 \ + -privacy-notice-url https://example.org/privacy \ + -confirmation-from 'TradeGravity ' +``` + +The form is `/tradegravity/subscribe` and the confirmation page is `/tradegravity/confirm`. A request creates a pending record for 30 minutes and sends one transactional email with a stable provider idempotency key. The token contains only a random ID. Neither signup nor confirmation responses disclose whether an address was active or globally suppressed. `GET` on the confirmation link is read-only; only a form-encoded `POST` activates the audience membership. An address that previously unsubscribed may re-consent through this flow, while a bounced, complaining, invalid, or provider-suppressed address cannot. Confirmation mail intentionally has no newsletter unsubscribe header because consent has not yet been completed. + +The built-in pending-record cap and one-minute resend cooldown are only backstops. Before exposing the form, apply IP/request-rate limits at the reverse proxy, prevent request-body/query logging, monitor aggregate failures, and consider a privacy-preserving challenge. Do not add `RESEND_API_KEY` to the static site or GitHub Actions. + +## Run the unsubscribe endpoint + +The Go process listens on loopback by default and expects a production TLS reverse proxy: + +```bash +export TRADEGRAVITY_UNSUBSCRIBE_SECRET='the-same-stable-secret' + +go run ./cmd/unsubscribe-service \ + -db private/subscriptions.db \ + -base-url https://subscriptions.example.org/tradegravity/ \ + -listen 127.0.0.1:8081 +``` + +For that base URL, the public endpoint is `/tradegravity/unsubscribe`; `/healthz` checks database readiness. + +The reverse proxy must: + +- terminate HTTPS and preserve the full path, query string, request body, and method; +- allow `GET` and form-encoded `POST` on the unsubscribe path; +- avoid logging the `token` query value or forwarding it to analytics; +- preserve `Cache-Control: no-store` and the supplied security headers; +- impose connection and request-rate limits without requiring cookies or login; and +- keep the service bound to a private interface. + +`GET` validates the token and displays a confirmation form but never changes subscription state, preventing ordinary link previews and scanners from silently unsubscribing a reader. A form-encoded `POST` with `List-Unsubscribe=One-Click` records `unsubscribed` once; repeated valid requests succeed without changing the original suppression time. No response includes the subscriber address. + +This matches the one-click body defined by [RFC 8058](https://www.rfc-editor.org/rfc/rfc8058.html). The pilot email sender puts the same URL in the visible body and `List-Unsubscribe` header and sets `List-Unsubscribe-Post: List-Unsubscribe=One-Click`; the configured provider/domain must ensure DKIM covers both headers. + +## Enable signed Resend feedback + +Register only the suppression-relevant Resend events—`email.bounced`, `email.complained`, and `email.suppressed`—against: + +```text +https://subscriptions.example.org/tradegravity/webhooks/resend +``` + +Store the endpoint signing secret separately, then enable the route explicitly: + +```bash +export TRADEGRAVITY_UNSUBSCRIBE_SECRET='the-same-stable-secret' +export RESEND_WEBHOOK_SECRET='whsec_read-from-your-secret-manager' + +go run ./cmd/unsubscribe-service \ + -db private/subscriptions.db \ + -base-url https://subscriptions.example.org/tradegravity/ \ + -listen 127.0.0.1:8081 \ + -enable-resend-webhook +``` + +The handler reads at most 64 KiB, verifies the signature against the untouched raw body and `svix-id`, `svix-timestamp`, and `svix-signature` headers, and only then parses the event. It expects one recipient because the pilot sender emits one provider request per recipient. Event IDs are stored without message bodies and deduplicated for at-least-once delivery. A verified event suppresses every active audience membership for the address and creates a global address suppression that blocks later consent imports. Replays return success without changing the original suppression. + +The reverse proxy must preserve those three signature headers and the raw body exactly, allow only POST on the feedback path, avoid request/body logging, and return the service response unchanged so Resend can retry failures. Signature verification follows [Resend's raw-body guidance](https://resend.com/docs/webhooks/verify-webhooks-requests); replay handling follows its documented `svix-id` at-least-once contract. + +## Current limits + +This is a single-instance reference service, not a hosted mailing platform. Before public use, provide and test: + +- production abuse protection for sign-up, confirmation, and unsubscribe traffic; +- encrypted backups, restore drills, retention, deletion, and operator-access audit; +- monitoring without query-token or address logging; +- a deployment plan for SQLite durability or a migration to a managed transactional store; +- production registration, secret rotation, replay drills, and monitoring for the signed feedback endpoint; and +- authenticated, idempotent launch authorization for delivery. + +Do not link the public TradeGravity site to this service until those controls, the privacy notice, verified sender, and public HTTPS deployment are ready. diff --git a/docs/USAGE.md b/docs/USAGE.md index 1262c95..50788db 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -79,3 +79,23 @@ For a network-free pipeline example, `cmd/sampledata` produces a deterministic S ## CSV and filtered views All explorer controls affect the treemaps and accessible table. **Download CSV** exports the filtered headline rows in raw source units; **Download filtered JSON** also records the active view state. JSON remains the canonical typed representation. See [DATA_SCHEMA.md](DATA_SCHEMA.md) and [DATA_RIGHTS.md](DATA_RIGHTS.md) before redistribution. + +## Reviewed email and social drafts + +The Chip Lens distribution desk reads `briefing.json` and shows three deterministic, cited observations. **Download email draft** exports Markdown with a link back to the evidence. **Download carousel copy** exports a six-slide, 4:5 JSON copy bundle whose slides retain evidence URLs. **Copy evidence link** returns readers to the Chip Lens rather than to an uncited claim. + +Both exports are editorial inputs, not automatic publishing instructions. Review the periods, values, source scope, wording, and rights before use. TradeGravity's static site does not collect email addresses, manage consent or unsubscribes, send mail, or publish to a social account. + +For an offline review package with mobile-first email HTML, a cited Instagram caption, two network-free native themes, six matched editable SVG originals and 1080×1350 PNG upload assets, alt text, file hashes, and an approval checklist, run `cmd/distributor` as documented in [DISTRIBUTION.md](DISTRIBUTION.md). Building that package still does not authorize delivery or social publication. + +After completing the checklist, use `cmd/distribution-approval` to verify the manifest and record a content release for `email`, `instagram`, or both. The record contains only a non-sensitive audience label and keeps consent, provider delivery, and automatic publishing false; it is not a subscriber list or a send instruction. + +For an Instagram-approved kit, run `cmd/instagram-preflight` outside the kit before opening a private platform draft. It rechecks all six PNG dimensions, caption evidence/scope/tags, alt-text completeness, manifest hashes, and channel approval while emitting only aggregate results with `manual_upload_required: true` and `automatic_publish_authorized: false`. See [INSTAGRAM_PREFLIGHT.md](INSTAGRAM_PREFLIGHT.md). + +For an email-approved kit, run `cmd/distribution-preflight` locally with private double-opt-in and suppression CSVs as documented in [EMAIL_DELIVERY_PREFLIGHT.md](EMAIL_DELIVERY_PREFLIGHT.md). Each active row must supply its own opaque HTTPS unsubscribe URL. The command applies suppressions, enforces URL uniqueness and the pilot limit, and writes an aggregate-only plan outside the kit without addresses or tokens. It never sends mail and deliberately leaves provider configuration and delivery authorization false. + +If the selected provider does not own consent and unsubscribe state, `cmd/subscription-registry` can import existing verified double-opt-in evidence into a separate private SQLite database and export the two preflight CSVs. The default-off signup mode in `cmd/unsubscribe-service` can instead create pending consent, send a short Resend confirmation, and activate only after the reader's explicit POST. The same service handles scanner-safe unsubscribe links and signed suppressions behind an HTTPS reverse proxy. Follow [UNSUBSCRIBE_SERVICE.md](UNSUBSCRIBE_SERVICE.md) before handling real addresses. + +For a deliberately bounded Resend pilot, `cmd/email-launch-approval` converts that exact aggregate preflight into a one-hour authorization only after the operator attests sender authentication, feedback suppression, privacy controls, and the final audience. `cmd/email-delivery` then reruns the live preflight, sends one isolated request per eligible recipient, and records only HMAC recipient identities in a private SQLite duplicate-prevention ledger. It remains inert without the matching files, `RESEND_API_KEY`, `TRADEGRAVITY_DELIVERY_SECRET`, and `-send-live`; see [EMAIL_PROVIDER_PILOT.md](EMAIL_PROVIDER_PILOT.md). + +If a provider response is uncertain, the ledger blocks every automatic retry. `cmd/email-delivery-reconcile` can record provider-confirmed acceptance or non-acceptance using the private recipient address only to derive the existing HMAC key. Acceptance becomes permanently skippable; confirmed non-acceptance can be retried only with a different short-lived launch authorization. Never delete the ledger to bypass this gate. diff --git a/examples/sample-data/README.md b/examples/sample-data/README.md index cc3d801..3ab201f 100644 --- a/examples/sample-data/README.md +++ b/examples/sample-data/README.md @@ -1,9 +1,11 @@ # Synthetic sample dataset -These files follow TradeGravity schema version 2.0 and contain three fictionalized reporter summaries with five annual periods, HS2 chapters, seven customs-visible semiconductor-stage proxies, 12 synthetic monthly periods, bilateral counterpart reports for mirror diagnostics, country context, quality signals, and evidence-grounded explanations. Values are synthetic and must not be used for research, policy, financial, or historical claims. The design/EDA stage remains context-only because services and intangible flows are not represented by the HS6 fixture. +These files follow TradeGravity schema version 2.0 and contain three fictionalized reporter summaries with five annual periods, HS2 chapters, seven customs-visible semiconductor-stage proxies, 12 synthetic monthly periods, a review-gated distribution briefing, bilateral counterpart reports for mirror diagnostics, country context, quality signals, and evidence-grounded explanations. Values are synthetic and must not be used for research, policy, financial, or historical claims. The design/EDA stage remains context-only because services and intangible flows are not represented by the HS6 fixture. The sample is intentionally small, deterministic, and network-independent. CI validates it with the same `cmd/validator` used before production deployment. +`briefing.json` also drives the synthetic distribution tests. CI builds both email and carousel assets, verifies the cited `caption.md`, records a temporary synthetic Instagram approval, and runs the aggregate manual-publish preflight. No social credential or external request is used. + To preview it, copy the entire directory contents—including `products/`, `strategic-hs6/`, `semiconductors/`, `bilateral-matrix/`, `mirror/`, and `explanations/`—into the ignored `site/data/` directory and start a static server as described in the root README. Regenerate the sample without network access: diff --git a/examples/sample-data/briefing.json b/examples/sample-data/briefing.json new file mode 100644 index 0000000..75864a2 --- /dev/null +++ b/examples/sample-data/briefing.json @@ -0,0 +1,190 @@ +{ + "schema_version": "1.0", + "generated_at": "2026-07-15T19:53:00Z", + "edition_id": "semiconductor-pulse-2023-12-20260715T195300Z", + "status": "ready", + "title": "TradeGravity Semiconductor Pulse", + "scope": "Deterministic distribution brief from selected monthly UN Comtrade HS6 observations against USA and China; not a complete semiconductor market, causal claim, or investment recommendation", + "latest_period": "2023-12", + "previous_period": "2023-11", + "publication_status": "baseline", + "review_required": true, + "signals": [ + { + "id": "monthly-total-kor", + "kind": "reporter_total_change", + "title": "Korea, Rep. selected chip trade increased", + "summary": "Selected monthly HS6 trade with USA and China moved from US$18.91B to US$19.21B (+1.6%).", + "reporter_iso3": "KOR", + "reporter_name": "Korea, Rep.", + "period": "2023-12", + "previous_period": "2023-11", + "current": { + "usa_trade_usd": 9550853736, + "china_trade_usd": 9662829262.560001, + "total_trade_usd": 19213682998.56, + "china_share": 0.502913952691121 + }, + "previous": { + "usa_trade_usd": 9371308266.666666, + "china_trade_usd": 9537108336, + "total_trade_usd": 18908416602.666664, + "china_share": 0.5043842927945101 + }, + "delta_trade_usd": 305266395.89333725, + "change_ratio": 0.016144471655563448, + "china_share_delta": -0.0014703401033890895, + "evidence": [ + "./semiconductors/monthly/KOR.json", + "./semiconductors/monthly/index.json" + ], + "interpretation": "A change in the selected two-anchor customs observations worth investigating; it does not establish production, demand, or causality.", + "measurement_limit": "Selected monthly HS6 observations against USA and China only; subject to source revisions and coverage limits." + }, + { + "id": "anchor-share-jpn", + "kind": "anchor_share_shift", + "title": "Japan two-anchor balance shifted toward USA", + "summary": "China's share of the selected USA-plus-China total moved from 50.4% to 50.3% (-0.1 percentage points).", + "reporter_iso3": "JPN", + "reporter_name": "Japan", + "period": "2023-12", + "previous_period": "2023-11", + "current": { + "usa_trade_usd": 8626577568.000002, + "china_trade_usd": 8727716753.280003, + "total_trade_usd": 17354294321.280006, + "china_share": 0.5029139526911209 + }, + "previous": { + "usa_trade_usd": 8464407466.666669, + "china_trade_usd": 8614162368.000002, + "total_trade_usd": 17078569834.666672, + "china_share": 0.50438429279451 + }, + "delta_trade_usd": 275724486.61333466, + "change_ratio": 0.01614447165556331, + "china_share_delta": -0.0014703401033890895, + "evidence": [ + "./semiconductors/monthly/JPN.json", + "./semiconductors/monthly/index.json" + ], + "interpretation": "The sign describes movement within the published USA-China anchor pair, not political alignment or global market share.", + "measurement_limit": "Selected monthly HS6 observations against USA and China only; subject to source revisions and coverage limits." + }, + { + "id": "product-kor-854231", + "kind": "product_total_change", + "title": "Korea, Rep. · Processors and controllers increased", + "summary": "HS6 854231 selected trade moved from US$6.14B to US$6.24B (+1.6%).", + "reporter_iso3": "KOR", + "reporter_name": "Korea, Rep.", + "classification": "H6", + "code": "854231", + "label": "Processors and controllers", + "period": "2023-12", + "previous_period": "2023-11", + "current": { + "usa_trade_usd": 3101765760, + "china_trade_usd": 3138131289.6000004, + "total_trade_usd": 6239897049.6, + "china_share": 0.502913952691121 + }, + "previous": { + "usa_trade_usd": 3043456000, + "china_trade_usd": 3097301760, + "total_trade_usd": 6140757760, + "china_share": 0.5043842927945101 + }, + "delta_trade_usd": 99139289.60000038, + "change_ratio": 0.0161444716555633, + "china_share_delta": -0.0014703401033890895, + "evidence": [ + "./semiconductors/monthly/KOR.json", + "./semiconductors/reference.json" + ], + "interpretation": "This is a product-level customs observation against USA and China, not company revenue, capacity, or a shipment route.", + "measurement_limit": "Selected monthly HS6 observations against USA and China only; subject to source revisions and coverage limits." + } + ], + "email": { + "subject": "TradeGravity Semiconductor Pulse · 2023-12", + "preview": "Three cited USA-China semiconductor observations for 2023-12; monthly movement is kept separate from publication revisions.", + "markdown": "# TradeGravity Semiconductor Pulse\n\nObservation window: 2023-12 vs 2023-11 · publication status: baseline\n\n## Korea, Rep. selected chip trade increased\n\nSelected monthly HS6 trade with USA and China moved from US$18.91B to US$19.21B (+1.6%).\n\nInterpretation boundary: A change in the selected two-anchor customs observations worth investigating; it does not establish production, demand, or causality.\n\n## Japan two-anchor balance shifted toward USA\n\nChina's share of the selected USA-plus-China total moved from 50.4% to 50.3% (-0.1 percentage points).\n\nInterpretation boundary: The sign describes movement within the published USA-China anchor pair, not political alignment or global market share.\n\n## Korea, Rep. · Processors and controllers increased\n\nHS6 854231 selected trade moved from US$6.14B to US$6.24B (+1.6%).\n\nInterpretation boundary: This is a product-level customs observation against USA and China, not company revenue, capacity, or a shipment route.\n\nReview the cited evidence before forwarding or publishing this draft.\n\n[Inspect the evidence]({{BASE_URL}}/?tab=semiconductors)\n\nData scope: selected monthly UN Comtrade HS6 observations against USA and China. Not investment, legal, or policy advice.", + "cta_label": "Inspect the evidence", + "cta_path": "./?tab=semiconductors", + "send_policy": "manual_review_required", + "primary_goal": "Return the reader to the cited TradeGravity evidence" + }, + "social_carousel": { + "format": "carousel_copy", + "aspect_ratio": "4:5", + "review_policy": "manual_review_required", + "slides": [ + { + "order": 1, + "role": "cover", + "headline": "TradeGravity Semiconductor Pulse", + "body": "Three USA-China semiconductor observations · 2023-12 vs 2023-11", + "evidence": [ + "./semiconductors/monthly/index.json" + ] + }, + { + "order": 2, + "role": "scale", + "headline": "Korea, Rep. selected chip trade increased", + "body": "Selected monthly HS6 trade with USA and China moved from US$18.91B to US$19.21B (+1.6%).", + "evidence": [ + "./semiconductors/monthly/KOR.json", + "./semiconductors/monthly/index.json" + ] + }, + { + "order": 3, + "role": "anchor_balance", + "headline": "Japan two-anchor balance shifted toward USA", + "body": "China's share of the selected USA-plus-China total moved from 50.4% to 50.3% (-0.1 percentage points).", + "evidence": [ + "./semiconductors/monthly/JPN.json", + "./semiconductors/monthly/index.json" + ] + }, + { + "order": 4, + "role": "product", + "headline": "Korea, Rep. · Processors and controllers increased", + "body": "HS6 854231 selected trade moved from US$6.14B to US$6.24B (+1.6%).", + "evidence": [ + "./semiconductors/monthly/KOR.json", + "./semiconductors/reference.json" + ] + }, + { + "order": 5, + "role": "method", + "headline": "Read the clocks separately", + "body": "Month-to-month customs movement and publish-to-publish revisions answer different questions. Neither proves causality or a physical route.", + "evidence": [ + "./changes.json", + "./semiconductors/monthly/index.json" + ] + }, + { + "order": 6, + "role": "cta", + "headline": "Inspect the evidence", + "body": "Open TradeGravity's Chip Lens for periods, values, sources, coverage, and limitations.", + "evidence": [ + "./?tab=semiconductors" + ] + } + ] + }, + "caveats": [ + "Monthly customs observations can be volatile, incomplete, and revised.", + "USA and China values are the two published anchor relationships, not world totals or physical shipment routes.", + "Publication-to-publication revisions are separate from economic month-to-month movement." + ], + "evidence_entry_point": "./?tab=semiconductors" +} diff --git a/examples/sample-data/catalog.json b/examples/sample-data/catalog.json index 7ec1795..e4fc63f 100644 --- a/examples/sample-data/catalog.json +++ b/examples/sample-data/catalog.json @@ -113,6 +113,17 @@ "partitioning": "single bounded change feed", "href": "./changes.json" }, + { + "id": "distribution_briefing", + "title": "Deterministic email and social briefing", + "status": "ready", + "provider": "tradegravity", + "classification": "source HS revision", + "product_level": 6, + "grain": "edition × selected monthly observation × distribution channel", + "partitioning": "single reviewed-draft publication", + "href": "./briefing.json" + }, { "id": "mirror_reconciliation", "title": "Unadjusted mirror-reporting diagnostics", diff --git a/go.mod b/go.mod index 813c686..12d9818 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,11 @@ module tradegravity go 1.25.12 -require modernc.org/sqlite v1.53.0 +require ( + github.com/svix/svix-webhooks v1.97.0 + golang.org/x/image v0.44.0 + modernc.org/sqlite v1.53.0 +) require ( github.com/dustin/go-humanize v1.0.1 // indirect @@ -11,6 +15,7 @@ require ( github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect golang.org/x/sys v0.44.0 // indirect + golang.org/x/text v0.40.0 // indirect modernc.org/libc v1.73.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index b054032..89ae80e 100644 --- a/go.sum +++ b/go.sum @@ -6,21 +6,29 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= +github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +github.com/svix/svix-webhooks v1.97.0 h1:vm2fM0/Clun0I5sY/35w+TiZV+DSlaDlAbMqfSOKkmg= +github.com/svix/svix-webhooks v1.97.0/go.mod h1:ngWxEvc1ll097e5kjOQfLz6PqjZTep6ORvGsV4s6mYg= +golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= +golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= diff --git a/internal/deliverypreflight/preflight.go b/internal/deliverypreflight/preflight.go new file mode 100644 index 0000000..1e78187 --- /dev/null +++ b/internal/deliverypreflight/preflight.go @@ -0,0 +1,543 @@ +package deliverypreflight + +import ( + "bytes" + "crypto/sha256" + "encoding/csv" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/mail" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "time" + "unicode" + + "tradegravity/internal/distributionkit" +) + +const ( + schemaVersion = "1.0" + toolVersion = "tradegravity-delivery-preflight/1.0" + maxSourceSize = 5 << 20 +) + +var subscriberHeader = []string{"email", "audience", "status", "consented_at", "consent_method", "consent_source", "privacy_notice_version", "unsubscribe_url"} +var suppressionHeader = []string{"email", "reason", "suppressed_at"} +var allowedSuppressionReasons = map[string]bool{ + "bounced": true, + "complaint": true, + "invalid": true, + "manual": true, + "unsubscribed": true, +} + +type Request struct { + KitDir string + SubscriberCSV string + SuppressionCSV string + GeneratedAt time.Time + MaxRecipients int +} + +type Plan struct { + SchemaVersion string `json:"schema_version"` + Tool string `json:"tool"` + Status string `json:"status"` + Channel string `json:"channel"` + EditionID string `json:"edition_id"` + ManifestSHA256 string `json:"manifest_sha256"` + ApprovalSHA256 string `json:"approval_sha256"` + Audience string `json:"audience"` + GeneratedAt string `json:"generated_at"` + Sources SourceProof `json:"sources"` + Counts Counts `json:"counts"` + MaxRecipients int `json:"max_recipients"` + ConsentValidated bool `json:"consent_validated"` + SuppressionApplied bool `json:"suppression_applied"` + UnsubscribeURLsValidated bool `json:"unsubscribe_urls_validated"` + ContainsRecipientAddresses bool `json:"contains_recipient_addresses"` + ProviderConfigured bool `json:"provider_configured"` + DeliveryAuthorized bool `json:"delivery_authorized"` + UnsubscribePlaceholder string `json:"unsubscribe_placeholder"` + RequiredProviderHeaders []string `json:"required_provider_headers"` + ListUnsubscribePostValue string `json:"list_unsubscribe_post_value"` + UnsubscribeHTTPSRequired bool `json:"unsubscribe_https_required"` + RequiredDKIMCoveredHeaders []string `json:"required_dkim_covered_headers"` + EmailTemplateSHA256 string `json:"email_template_sha256"` +} + +type SourceProof struct { + SubscriberCSVSHA256 string `json:"subscriber_csv_sha256"` + SuppressionCSVSHA256 string `json:"suppression_csv_sha256"` +} + +type Counts struct { + Consented int `json:"consented"` + Suppressed int `json:"suppressed"` + SuppressionRows int `json:"suppression_rows"` + Eligible int `json:"eligible"` +} + +type Result struct { + Plan Plan + EligibleRecipients []Recipient + JSON []byte +} + +type Recipient struct { + Email string + UnsubscribeURL string +} + +func Build(request Request) (Result, error) { + if request.GeneratedAt.IsZero() { + return Result{}, errors.New("preflight generation time is required") + } + if request.MaxRecipients < 1 || request.MaxRecipients > 1000 { + return Result{}, errors.New("max recipients must be between 1 and 1000") + } + kitRoot, err := canonicalDirectory(request.KitDir) + if err != nil { + return Result{}, err + } + approval, manifest, err := distributionkit.VerifyApproved(kitRoot, "email") + if err != nil { + return Result{}, fmt.Errorf("verify approved email kit: %w", err) + } + + subscriberRaw, subscriberPath, err := readExternalSource(kitRoot, request.SubscriberCSV, "subscriber CSV") + if err != nil { + return Result{}, err + } + suppressionRaw, suppressionPath, err := readExternalSource(kitRoot, request.SuppressionCSV, "suppression CSV") + if err != nil { + return Result{}, err + } + if samePath(subscriberPath, suppressionPath) { + return Result{}, errors.New("subscriber and suppression CSVs must be different files") + } + + consented, err := parseSubscribers(subscriberRaw, approval.Audience, request.GeneratedAt) + if err != nil { + return Result{}, err + } + suppressed, suppressionRows, err := parseSuppressions(suppressionRaw, request.GeneratedAt) + if err != nil { + return Result{}, err + } + eligible := make([]Recipient, 0, len(consented)) + matchedSuppressions := 0 + for _, recipient := range consented { + if suppressed[recipient.Email] { + matchedSuppressions++ + continue + } + eligible = append(eligible, recipient) + } + if len(eligible) == 0 { + return Result{}, errors.New("no eligible recipients remain after suppression") + } + if len(eligible) > request.MaxRecipients { + return Result{}, fmt.Errorf("eligible recipient count %d exceeds pilot limit %d", len(eligible), request.MaxRecipients) + } + sort.Slice(eligible, func(first, second int) bool { + return eligible[first].Email < eligible[second].Email + }) + + templateRaw, templateDigest, err := verifiedEmailTemplate(kitRoot, manifest) + if err != nil { + return Result{}, err + } + if bytes.Count(templateRaw, []byte("{{UNSUBSCRIBE_URL}}")) != 1 { + return Result{}, errors.New("approved email template must contain exactly one unsubscribe placeholder") + } + markdownRaw, err := os.ReadFile(filepath.Join(kitRoot, "email", "body.md")) + if err != nil { + return Result{}, fmt.Errorf("read approved email Markdown: %w", err) + } + if bytes.Count(markdownRaw, []byte("{{UNSUBSCRIBE_URL}}")) != 1 { + return Result{}, errors.New("approved email Markdown must contain exactly one unsubscribe placeholder") + } + lowerTemplate := bytes.ToLower(templateRaw) + if bytes.Contains(lowerTemplate, []byte(" unicode.MaxASCII || unicode.IsSpace(r) || unicode.IsControl(r) { + return "", errors.New("email address must use a plain ASCII addr-spec") + } + } + parsed, err := mail.ParseAddress(value) + if err != nil || parsed.Address != value { + return "", errors.New("email address must not contain a display name or invalid syntax") + } + parts := strings.Split(value, "@") + if parts[0] == "" || parts[1] == "" || !strings.Contains(parts[1], ".") || strings.HasPrefix(parts[1], ".") || strings.HasSuffix(parts[1], ".") { + return "", errors.New("email address must contain a valid-looking domain") + } + return strings.ToLower(value), nil +} + +func validateTimestamp(value string, generatedAt time.Time, kind string, line int) error { + parsed, err := time.Parse(time.RFC3339, value) + if err != nil { + return fmt.Errorf("CSV line %d has invalid %s timestamp", line, kind) + } + if parsed.After(generatedAt) { + return fmt.Errorf("CSV line %d has future %s timestamp", line, kind) + } + return nil +} + +func validateLabel(value, name string, line int) error { + if value == "" || len([]rune(value)) > 120 { + return fmt.Errorf("CSV line %d has invalid %s", line, name) + } + for _, r := range value { + if unicode.IsControl(r) { + return fmt.Errorf("CSV line %d has invalid %s", line, name) + } + } + return nil +} + +func verifiedEmailTemplate(kitRoot string, manifest distributionkit.Manifest) ([]byte, string, error) { + var declared *distributionkit.ManifestFile + for index := range manifest.Files { + if manifest.Files[index].Path == "email/body.html" { + declared = &manifest.Files[index] + break + } + } + if declared == nil { + return nil, "", errors.New("approved manifest does not contain email/body.html") + } + raw, err := os.ReadFile(filepath.Join(kitRoot, "email", "body.html")) + if err != nil { + return nil, "", fmt.Errorf("read approved email template: %w", err) + } + digest := sha256.Sum256(raw) + value := hex.EncodeToString(digest[:]) + if value != declared.SHA256 { + return nil, "", errors.New("approved email template digest does not match manifest") + } + return raw, value, nil +} + +func readExternalSource(kitRoot, path, label string) ([]byte, string, error) { + path = strings.TrimSpace(path) + if path == "" { + return nil, "", fmt.Errorf("%s path is required", label) + } + absolute, err := filepath.Abs(path) + if err != nil { + return nil, "", fmt.Errorf("resolve %s: %w", label, err) + } + originalInfo, err := os.Lstat(absolute) + if err != nil { + return nil, "", fmt.Errorf("inspect %s: %w", label, err) + } + if originalInfo.Mode()&os.ModeSymlink != 0 { + return nil, "", fmt.Errorf("%s must be a regular non-symlink file", label) + } + resolved, err := filepath.EvalSymlinks(absolute) + if err != nil { + return nil, "", fmt.Errorf("resolve %s links: %w", label, err) + } + if pathWithin(kitRoot, resolved) { + return nil, "", fmt.Errorf("%s must stay outside the distribution kit", label) + } + info, err := os.Lstat(resolved) + if err != nil { + return nil, "", fmt.Errorf("inspect %s: %w", label, err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return nil, "", fmt.Errorf("%s must be a regular non-symlink file", label) + } + if info.Size() > maxSourceSize { + return nil, "", fmt.Errorf("%s exceeds the %d-byte limit", label, maxSourceSize) + } + raw, err := os.ReadFile(resolved) + if err != nil { + return nil, "", fmt.Errorf("read %s: %w", label, err) + } + return raw, resolved, nil +} + +func canonicalDirectory(path string) (string, error) { + path = strings.TrimSpace(path) + if path == "" { + return "", errors.New("distribution-kit directory is required") + } + absolute, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("resolve distribution-kit directory: %w", err) + } + resolved, err := filepath.EvalSymlinks(absolute) + if err != nil { + return "", fmt.Errorf("resolve distribution-kit links: %w", err) + } + info, err := os.Stat(resolved) + if err != nil || !info.IsDir() { + return "", errors.New("distribution-kit path must be a directory") + } + return resolved, nil +} + +func pathWithin(root, candidate string) bool { + relative, err := filepath.Rel(root, candidate) + if err != nil { + return false + } + return relative == "." || (relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))) +} + +func samePath(first, second string) bool { + return strings.EqualFold(filepath.Clean(first), filepath.Clean(second)) +} + +func equalStrings(first, second []string) bool { + if len(first) != len(second) { + return false + } + for index := range first { + if first[index] != second[index] { + return false + } + } + return true +} diff --git a/internal/deliverypreflight/preflight_test.go b/internal/deliverypreflight/preflight_test.go new file mode 100644 index 0000000..fb5dd09 --- /dev/null +++ b/internal/deliverypreflight/preflight_test.go @@ -0,0 +1,201 @@ +package deliverypreflight + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "tradegravity/internal/distributionkit" +) + +func TestBuildProducesAggregateOnlyConsentPreflight(t *testing.T) { + root, kitDir := buildApprovedKit(t) + subscribers := writeFixture(t, root, "subscribers.csv", strings.Join([]string{ + "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url", + "alpha@example.invalid,pilot-audience,active,2026-07-10T01:00:00Z,double_opt_in,website-form,v1,https://subscriptions.example.invalid/u/opaque-alpha-token", + "beta@example.invalid,pilot-audience,active,2026-07-11T02:00:00Z,double_opt_in,website-form,v1,https://subscriptions.example.invalid/u/opaque-beta-token", + "", + }, "\n")) + suppressions := writeFixture(t, root, "suppressions.csv", strings.Join([]string{ + "email,reason,suppressed_at", + "beta@example.invalid,unsubscribed,2026-07-12T03:00:00Z", + "old@example.invalid,bounced,2026-07-01T00:00:00Z", + "", + }, "\n")) + + result, err := Build(Request{ + KitDir: kitDir, + SubscriberCSV: subscribers, + SuppressionCSV: suppressions, + GeneratedAt: time.Date(2026, 7, 17, 12, 0, 0, 0, time.FixedZone("KST", 9*60*60)), + MaxRecipients: 25, + }) + if err != nil { + t.Fatalf("Build() error = %v", err) + } + if result.Plan.Status != "consent_preflight_passed" || result.Plan.Channel != "email" || result.Plan.Audience != "pilot-audience" { + t.Fatalf("unexpected preflight identity: %+v", result.Plan) + } + if result.Plan.Counts.Consented != 2 || result.Plan.Counts.Suppressed != 1 || result.Plan.Counts.SuppressionRows != 2 || result.Plan.Counts.Eligible != 1 { + t.Fatalf("unexpected preflight counts: %+v", result.Plan.Counts) + } + if !result.Plan.ConsentValidated || !result.Plan.SuppressionApplied || !result.Plan.UnsubscribeURLsValidated || result.Plan.ContainsRecipientAddresses || result.Plan.ProviderConfigured || result.Plan.DeliveryAuthorized { + t.Fatalf("preflight safety state is wrong: %+v", result.Plan) + } + if strings.Join(result.Plan.RequiredProviderHeaders, ",") != "List-Unsubscribe,List-Unsubscribe-Post" || result.Plan.ListUnsubscribePostValue != "List-Unsubscribe=One-Click" || !result.Plan.UnsubscribeHTTPSRequired || strings.Join(result.Plan.RequiredDKIMCoveredHeaders, ",") != "List-Unsubscribe,List-Unsubscribe-Post" || len(result.Plan.EmailTemplateSHA256) != 64 { + t.Fatalf("unsubscribe contract is incomplete: %+v", result.Plan) + } + if len(result.EligibleRecipients) != 1 || result.EligibleRecipients[0].Email != "alpha@example.invalid" || result.EligibleRecipients[0].UnsubscribeURL != "https://subscriptions.example.invalid/u/opaque-alpha-token" { + t.Fatalf("unexpected in-memory eligible recipients: %v", result.EligibleRecipients) + } + for _, forbidden := range []string{"alpha@example.invalid", "beta@example.invalid", "opaque-alpha-token", "opaque-beta-token", subscribers, suppressions} { + if bytes.Contains(result.JSON, []byte(forbidden)) { + t.Fatalf("aggregate plan leaked recipient PII or a local path: %s", forbidden) + } + } + + output := filepath.Join(root, "delivery-preflight.json") + if err := Write(output, kitDir, result.JSON); err != nil { + t.Fatalf("Write() error = %v", err) + } + if err := Write(output, kitDir, result.JSON); err == nil { + t.Fatal("Write() overwrote an existing preflight") + } + if err := Write(filepath.Join(kitDir, "delivery-preflight.json"), kitDir, result.JSON); err == nil { + t.Fatal("Write() placed recipient-source metadata inside the approved kit") + } +} + +func TestBuildFailsClosedForConsentAudienceSuppressionAndLimit(t *testing.T) { + root, kitDir := buildApprovedKit(t) + emptySuppressions := writeFixture(t, root, "empty-suppressions.csv", "email,reason,suppressed_at\n") + generatedAt := time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC) + + tests := []struct { + name string + subscribers string + suppressions string + max int + want string + }{ + { + name: "wrong audience", + subscribers: "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url\nalpha@example.invalid,another-audience,active,2026-07-10T00:00:00Z,double_opt_in,form,v1,https://subscriptions.example.invalid/u/wrong-audience-token\n", + suppressions: emptySuppressions, + max: 25, + want: "audience does not match", + }, + { + name: "single opt in", + subscribers: "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url\nalpha@example.invalid,pilot-audience,active,2026-07-10T00:00:00Z,single_opt_in,form,v1,https://subscriptions.example.invalid/u/single-token\n", + suppressions: emptySuppressions, + max: 25, + want: "not active double opt-in", + }, + { + name: "pilot limit", + subscribers: "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url\nalpha@example.invalid,pilot-audience,active,2026-07-10T00:00:00Z,double_opt_in,form,v1,https://subscriptions.example.invalid/u/limit-alpha-token\nbeta@example.invalid,pilot-audience,active,2026-07-10T00:00:00Z,double_opt_in,form,v1,https://subscriptions.example.invalid/u/limit-beta-token\n", + suppressions: emptySuppressions, + max: 1, + want: "exceeds pilot limit", + }, + { + name: "fully suppressed", + subscribers: "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url\nalpha@example.invalid,pilot-audience,active,2026-07-10T00:00:00Z,double_opt_in,form,v1,https://subscriptions.example.invalid/u/suppressed-token\n", + suppressions: "email,reason,suppressed_at\nalpha@example.invalid,complaint,2026-07-11T00:00:00Z\n", + max: 25, + want: "no eligible recipients", + }, + { + name: "insecure unsubscribe URL", + subscribers: "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url\nalpha@example.invalid,pilot-audience,active,2026-07-10T00:00:00Z,double_opt_in,form,v1,http://subscriptions.example.invalid/u/insecure-token\n", + suppressions: emptySuppressions, + max: 25, + want: "must be absolute HTTPS", + }, + { + name: "address in unsubscribe URL", + subscribers: "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url\nalpha@example.invalid,pilot-audience,active,2026-07-10T00:00:00Z,double_opt_in,form,v1,https://subscriptions.example.invalid/u/alpha%40example.invalid\n", + suppressions: emptySuppressions, + max: 25, + want: "must not expose an email address", + }, + { + name: "duplicate unsubscribe URL", + subscribers: "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url\nalpha@example.invalid,pilot-audience,active,2026-07-10T00:00:00Z,double_opt_in,form,v1,https://subscriptions.example.invalid/u/shared-token\nbeta@example.invalid,pilot-audience,active,2026-07-10T00:00:00Z,double_opt_in,form,v1,https://subscriptions.example.invalid/u/shared-token\n", + suppressions: emptySuppressions, + max: 25, + want: "duplicates an unsubscribe URL", + }, + { + name: "future consent", + subscribers: "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url\nalpha@example.invalid,pilot-audience,active,2026-07-18T00:00:00Z,double_opt_in,form,v1,https://subscriptions.example.invalid/u/future-token\n", + suppressions: emptySuppressions, + max: 25, + want: "future consent timestamp", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + subscriberPath := writeFixture(t, root, strings.ReplaceAll(test.name, " ", "-")+"-subscribers.csv", test.subscribers) + suppressionPath := test.suppressions + if !filepath.IsAbs(suppressionPath) { + suppressionPath = writeFixture(t, root, strings.ReplaceAll(test.name, " ", "-")+"-suppressions.csv", test.suppressions) + } + _, err := Build(Request{KitDir: kitDir, SubscriberCSV: subscriberPath, SuppressionCSV: suppressionPath, GeneratedAt: generatedAt, MaxRecipients: test.max}) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("Build() error = %v, want %q", err, test.want) + } + }) + } + + insideKit := filepath.Join(kitDir, "email", "body.md") + _, err := Build(Request{KitDir: kitDir, SubscriberCSV: insideKit, SuppressionCSV: emptySuppressions, GeneratedAt: generatedAt, MaxRecipients: 25}) + if err == nil || !strings.Contains(err.Error(), "outside the distribution kit") { + t.Fatalf("Build() inside-kit source error = %v", err) + } +} + +func buildApprovedKit(t *testing.T) (string, string) { + t.Helper() + root := t.TempDir() + briefingPath := filepath.Join("..", "..", "examples", "sample-data", "briefing.json") + briefing, err := os.ReadFile(briefingPath) + if err != nil { + t.Fatal(err) + } + bundle, err := distributionkit.Build(briefing, "https://example.org/TradeGravity/") + if err != nil { + t.Fatal(err) + } + kitDir := filepath.Join(root, "kit") + if err := distributionkit.Write(kitDir, bundle); err != nil { + t.Fatal(err) + } + _, approvalRaw, err := distributionkit.Approve(kitDir, distributionkit.ApprovalRequest{ + Reviewer: "reviewer", + Audience: "pilot-audience", + Channels: []string{"email"}, + ApprovedAt: time.Date(2026, 7, 17, 0, 0, 0, 0, time.UTC), + Attested: true, + }) + if err != nil { + t.Fatal(err) + } + if err := distributionkit.WriteApproval(kitDir, approvalRaw); err != nil { + t.Fatal(err) + } + return root, kitDir +} + +func writeFixture(t *testing.T, root, name, content string) string { + t.Helper() + path := filepath.Join(root, name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/internal/distributionkit/approval.go b/internal/distributionkit/approval.go new file mode 100644 index 0000000..d69dc3e --- /dev/null +++ b/internal/distributionkit/approval.go @@ -0,0 +1,431 @@ +package distributionkit + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + approvalSchemaVersion = "1.0" + approvalScope = "content_release" +) + +var allowedApprovalChannels = map[string]bool{ + "email": true, + "instagram": true, +} + +var contentApprovalAttestations = []string{ + "evidence_periods_values_and_units_reviewed", + "claims_caveats_links_and_rights_reviewed", + "final_channel_assets_and_alt_text_reviewed", +} + +type ApprovalRequest struct { + Reviewer string + Audience string + Channels []string + ApprovedAt time.Time + Attested bool +} + +type Approval struct { + SchemaVersion string `json:"schema_version"` + Tool string `json:"tool"` + Scope string `json:"scope"` + Status string `json:"status"` + EditionID string `json:"edition_id"` + ManifestSHA256 string `json:"manifest_sha256"` + ManifestFileCount int `json:"manifest_file_count"` + ApprovedAt string `json:"approved_at"` + Reviewer string `json:"reviewer"` + Audience string `json:"audience"` + Channels []string `json:"channels"` + Attestations []string `json:"attestations"` + ProviderDeliveryReady bool `json:"provider_delivery_ready"` + SubscriberConsentReady bool `json:"subscriber_consent_ready"` + AutomaticPublishReady bool `json:"automatic_publish_ready"` +} + +func Approve(kitDir string, request ApprovalRequest) (Approval, []byte, error) { + if err := validateApprovalRequest(request); err != nil { + return Approval{}, nil, err + } + manifest, manifestRaw, err := Verify(kitDir) + if err != nil { + return Approval{}, nil, err + } + digest := sha256.Sum256(manifestRaw) + channels := append([]string(nil), request.Channels...) + sort.Strings(channels) + approval := Approval{ + SchemaVersion: approvalSchemaVersion, + Tool: kitToolVersion, + Scope: approvalScope, + Status: "approved", + EditionID: manifest.EditionID, + ManifestSHA256: hex.EncodeToString(digest[:]), + ManifestFileCount: len(manifest.Files), + ApprovedAt: request.ApprovedAt.UTC().Format(time.RFC3339), + Reviewer: strings.TrimSpace(request.Reviewer), + Audience: strings.TrimSpace(request.Audience), + Channels: channels, + Attestations: append([]string(nil), contentApprovalAttestations...), + ProviderDeliveryReady: false, + SubscriberConsentReady: false, + AutomaticPublishReady: false, + } + raw, err := json.MarshalIndent(approval, "", " ") + if err != nil { + return Approval{}, nil, fmt.Errorf("encode approval: %w", err) + } + return approval, append(raw, '\n'), nil +} + +func Verify(kitDir string) (Manifest, []byte, error) { + return verifyKit(kitDir, false) +} + +func VerifyApproved(kitDir, requiredChannel string) (Approval, Manifest, error) { + requiredChannel = strings.TrimSpace(requiredChannel) + if !allowedApprovalChannels[requiredChannel] { + return Approval{}, Manifest{}, fmt.Errorf("unsupported required channel %q", requiredChannel) + } + root, err := cleanKitRoot(kitDir) + if err != nil { + return Approval{}, Manifest{}, err + } + manifest, manifestRaw, err := verifyKit(root, true) + if err != nil { + return Approval{}, Manifest{}, err + } + approvalRaw, err := os.ReadFile(filepath.Join(root, "approval.json")) + if err != nil { + return Approval{}, Manifest{}, fmt.Errorf("read approval.json: %w", err) + } + var approval Approval + decoder := json.NewDecoder(bytes.NewReader(approvalRaw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&approval); err != nil { + return Approval{}, Manifest{}, fmt.Errorf("decode approval.json: %w", err) + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return Approval{}, Manifest{}, errors.New("approval.json contains trailing JSON values") + } + canonicalApproval, err := marshalCanonicalJSON(approval) + if err != nil { + return Approval{}, Manifest{}, fmt.Errorf("canonicalize approval.json: %w", err) + } + if !bytes.Equal(approvalRaw, canonicalApproval) { + return Approval{}, Manifest{}, errors.New("approval.json must use the canonical generated encoding") + } + if err := validateStoredApproval(approval, manifest, manifestRaw, requiredChannel); err != nil { + return Approval{}, Manifest{}, err + } + return approval, manifest, nil +} + +func verifyKit(kitDir string, allowApproval bool) (Manifest, []byte, error) { + root, err := cleanKitRoot(kitDir) + if err != nil { + return Manifest{}, nil, err + } + manifestPath := filepath.Join(root, "manifest.json") + manifestInfo, err := os.Lstat(manifestPath) + if err != nil { + return Manifest{}, nil, fmt.Errorf("inspect manifest: %w", err) + } + if !manifestInfo.Mode().IsRegular() { + return Manifest{}, nil, errors.New("manifest.json must be a regular file") + } + manifestRaw, err := os.ReadFile(manifestPath) + if err != nil { + return Manifest{}, nil, fmt.Errorf("read manifest: %w", err) + } + var manifest Manifest + decoder := json.NewDecoder(bytes.NewReader(manifestRaw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&manifest); err != nil { + return Manifest{}, nil, fmt.Errorf("decode manifest: %w", err) + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return Manifest{}, nil, errors.New("manifest contains trailing JSON values") + } + canonicalManifest, err := marshalCanonicalJSON(manifest) + if err != nil { + return Manifest{}, nil, fmt.Errorf("canonicalize manifest: %w", err) + } + if !bytes.Equal(manifestRaw, canonicalManifest) { + return Manifest{}, nil, errors.New("manifest.json must use the canonical generated encoding") + } + if err := validateApprovalManifest(manifest); err != nil { + return Manifest{}, nil, err + } + + expected := map[string]ManifestFile{"manifest.json": {Path: "manifest.json"}} + if allowApproval { + expected["approval.json"] = ManifestFile{Path: "approval.json"} + } + previousPath := "" + for _, item := range manifest.Files { + if !fs.ValidPath(item.Path) || strings.Contains(item.Path, "\\") || item.Path == "." || item.Path == "manifest.json" || item.Path == "approval.json" { + return Manifest{}, nil, fmt.Errorf("manifest contains invalid file path %q", item.Path) + } + if item.Path <= previousPath { + return Manifest{}, nil, errors.New("manifest files must be unique and sorted by path") + } + previousPath = item.Path + if item.MediaType != mediaType(item.Path) || item.Bytes < 0 || len(item.SHA256) != sha256.Size*2 { + return Manifest{}, nil, fmt.Errorf("manifest metadata is invalid for %s", item.Path) + } + if _, err := hex.DecodeString(item.SHA256); err != nil { + return Manifest{}, nil, fmt.Errorf("manifest digest is invalid for %s", item.Path) + } + expected[item.Path] = item + } + + seen := map[string]bool{} + err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if path == root { + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("distribution kit must not contain symlinks: %s", entry.Name()) + } + if entry.IsDir() { + return nil + } + if !entry.Type().IsRegular() { + return fmt.Errorf("distribution kit contains a non-regular file: %s", entry.Name()) + } + relative, relErr := filepath.Rel(root, path) + if relErr != nil { + return relErr + } + relative = filepath.ToSlash(relative) + item, ok := expected[relative] + if !ok { + return fmt.Errorf("distribution kit contains untracked file %q", relative) + } + seen[relative] = true + if relative == "manifest.json" || relative == "approval.json" { + return nil + } + content, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + if len(content) != item.Bytes { + return fmt.Errorf("distribution file size changed: %s", relative) + } + digest := sha256.Sum256(content) + if !strings.EqualFold(hex.EncodeToString(digest[:]), item.SHA256) { + return fmt.Errorf("distribution file digest changed: %s", relative) + } + return nil + }) + if err != nil { + return Manifest{}, nil, fmt.Errorf("verify distribution kit: %w", err) + } + if len(seen) != len(expected) { + missing := make([]string, 0) + for path := range expected { + if !seen[path] { + missing = append(missing, path) + } + } + sort.Strings(missing) + return Manifest{}, nil, fmt.Errorf("distribution kit is missing files: %s", strings.Join(missing, ", ")) + } + return manifest, manifestRaw, nil +} + +func WriteApproval(kitDir string, content []byte) error { + root, err := cleanKitRoot(kitDir) + if err != nil { + return err + } + target := filepath.Join(root, "approval.json") + file, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + if errors.Is(err, fs.ErrExist) { + return errors.New("approval.json already exists; rebuild the kit for a new approval") + } + return fmt.Errorf("create approval.json: %w", err) + } + writeErr := func() error { + if _, err := file.Write(content); err != nil { + return err + } + return file.Sync() + }() + closeErr := file.Close() + if writeErr != nil { + _ = os.Remove(target) + return fmt.Errorf("write approval.json: %w", writeErr) + } + if closeErr != nil { + _ = os.Remove(target) + return fmt.Errorf("close approval.json: %w", closeErr) + } + return nil +} + +func validateApprovalRequest(request ApprovalRequest) error { + if !request.Attested { + return errors.New("explicit review attestation is required") + } + if err := validateApprovalLabel("reviewer", request.Reviewer, 100); err != nil { + return err + } + if err := validateApprovalAudience(request.Audience); err != nil { + return err + } + if request.ApprovedAt.IsZero() { + return errors.New("approval time is required") + } + if len(request.Channels) == 0 { + return errors.New("at least one approval channel is required") + } + seen := map[string]bool{} + for _, channel := range request.Channels { + if channel != strings.TrimSpace(channel) || !allowedApprovalChannels[channel] { + return fmt.Errorf("unsupported approval channel %q", channel) + } + if seen[channel] { + return fmt.Errorf("duplicate approval channel %q", channel) + } + seen[channel] = true + } + return nil +} + +func validateApprovalLabel(name, value string, limit int) error { + trimmed := strings.TrimSpace(value) + if trimmed == "" || len([]rune(trimmed)) > limit { + return fmt.Errorf("%s must contain 1-%d characters", name, limit) + } + for _, r := range trimmed { + if r < 0x20 || r == 0x7f { + return fmt.Errorf("%s must not contain control characters", name) + } + } + return nil +} + +func validateApprovalAudience(value string) error { + if err := validateApprovalLabel("audience", value, 120); err != nil { + return err + } + if strings.ContainsAny(value, "@/\\") || strings.Contains(value, "://") { + return errors.New("audience must be a non-sensitive label, not an address or path") + } + return nil +} + +func validateApprovalManifest(manifest Manifest) error { + if manifest.SchemaVersion != kitSchemaVersion || manifest.Tool != kitToolVersion || !safeIdentifier(manifest.EditionID) { + return errors.New("manifest provenance does not match the distribution-kit contract") + } + if manifest.DistributionStatus != "review_pending" || !manifest.ReviewRequired || manifest.SendAuthorized || manifest.SocialPublishAuthorized { + return errors.New("only an unchanged review-pending kit can be approved") + } + if len(manifest.Files) == 0 || manifest.Carousel.Width != cardWidth || manifest.Carousel.Height != cardHeight || manifest.Carousel.SlideCount != 6 { + return errors.New("manifest content contract is incomplete") + } + if _, err := resolveTheme(manifest.Carousel.Theme); err != nil || strings.TrimSpace(manifest.Carousel.Theme) == "" { + return errors.New("manifest carousel theme is unsupported") + } + if manifest.Carousel.CaptionPath != "carousel/caption.md" { + return errors.New("manifest carousel caption contract is incomplete") + } + if strings.Join(manifest.Carousel.Formats, ",") != "png,svg" { + return errors.New("manifest must contain matched PNG and SVG carousel assets") + } + return nil +} + +func validateStoredApproval(approval Approval, manifest Manifest, manifestRaw []byte, requiredChannel string) error { + if approval.SchemaVersion != approvalSchemaVersion || approval.Tool != kitToolVersion || approval.Scope != approvalScope || approval.Status != "approved" { + return errors.New("approval.json does not match the content-release contract") + } + if approval.EditionID != manifest.EditionID || approval.ManifestFileCount != len(manifest.Files) { + return errors.New("approval.json does not match the manifest edition") + } + digest := sha256.Sum256(manifestRaw) + if approval.ManifestSHA256 != hex.EncodeToString(digest[:]) { + return errors.New("approval.json manifest digest does not match") + } + if err := validateApprovalLabel("reviewer", approval.Reviewer, 100); err != nil { + return fmt.Errorf("invalid stored approval: %w", err) + } + if err := validateApprovalAudience(approval.Audience); err != nil { + return fmt.Errorf("invalid stored approval: %w", err) + } + approvedAt, err := time.Parse(time.RFC3339, approval.ApprovedAt) + if err != nil || approvedAt.UTC().Format(time.RFC3339) != approval.ApprovedAt { + return errors.New("approval.json must contain a canonical UTC approval time") + } + if strings.Join(approval.Attestations, "\n") != strings.Join(contentApprovalAttestations, "\n") { + return errors.New("approval.json attestations do not match the content-release contract") + } + if approval.ProviderDeliveryReady || approval.SubscriberConsentReady || approval.AutomaticPublishReady { + return errors.New("content approval must not claim provider, consent, or automatic-publish readiness") + } + foundRequired := false + previous := "" + for _, channel := range approval.Channels { + if !allowedApprovalChannels[channel] || channel <= previous { + return errors.New("approval.json channels must be unique, supported, and sorted") + } + previous = channel + if channel == requiredChannel { + foundRequired = true + } + } + if !foundRequired { + return fmt.Errorf("content is not approved for channel %q", requiredChannel) + } + return nil +} + +func cleanKitRoot(kitDir string) (string, error) { + kitDir = strings.TrimSpace(kitDir) + if kitDir == "" { + return "", errors.New("distribution-kit directory is required") + } + root, err := filepath.Abs(kitDir) + if err != nil { + return "", fmt.Errorf("resolve distribution-kit directory: %w", err) + } + info, err := os.Stat(root) + if err != nil { + return "", fmt.Errorf("inspect distribution-kit directory: %w", err) + } + if !info.IsDir() { + return "", errors.New("distribution-kit path must be a directory") + } + return root, nil +} + +func marshalCanonicalJSON(value any) ([]byte, error) { + raw, err := json.MarshalIndent(value, "", " ") + if err != nil { + return nil, err + } + return append(raw, '\n'), nil +} diff --git a/internal/distributionkit/approval_test.go b/internal/distributionkit/approval_test.go new file mode 100644 index 0000000..ab3eaff --- /dev/null +++ b/internal/distributionkit/approval_test.go @@ -0,0 +1,189 @@ +package distributionkit + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestApproveBindsReviewedContentWithoutClaimingDeliveryReadiness(t *testing.T) { + kitDir := buildWrittenKit(t) + request := ApprovalRequest{ + Reviewer: "maintainer-handle", + Audience: "consented-internal-pilot", + Channels: []string{"instagram", "email"}, + ApprovedAt: time.Date(2026, 7, 17, 12, 34, 56, 0, time.FixedZone("KST", 9*60*60)), + Attested: true, + } + approval, raw, err := Approve(kitDir, request) + if err != nil { + t.Fatalf("Approve() error = %v", err) + } + if approval.Scope != "content_release" || approval.Status != "approved" || strings.Join(approval.Channels, ",") != "email,instagram" { + t.Fatalf("unexpected approval contract: %+v", approval) + } + if approval.ProviderDeliveryReady || approval.SubscriberConsentReady || approval.AutomaticPublishReady { + t.Fatalf("content approval overclaims delivery readiness: %+v", approval) + } + if approval.ApprovedAt != "2026-07-17T03:34:56Z" || len(approval.ManifestSHA256) != 64 || approval.ManifestFileCount != 20 { + t.Fatalf("approval provenance is incomplete: %+v", approval) + } + if len(approval.Attestations) != 3 || !bytes.HasSuffix(raw, []byte("\n")) { + t.Fatalf("approval attestation serialization is incomplete: %s", raw) + } + if err := WriteApproval(kitDir, raw); err != nil { + t.Fatalf("WriteApproval() error = %v", err) + } + verified, verifiedManifest, err := VerifyApproved(kitDir, "email") + if err != nil { + t.Fatalf("VerifyApproved() error = %v", err) + } + if verified.ManifestSHA256 != approval.ManifestSHA256 || verifiedManifest.EditionID != approval.EditionID { + t.Fatal("VerifyApproved() did not preserve the approval-manifest binding") + } + if _, _, err := VerifyApproved(kitDir, "instagram"); err != nil { + t.Fatalf("VerifyApproved() rejected an approved channel: %v", err) + } + if err := WriteApproval(kitDir, raw); err == nil { + t.Fatal("WriteApproval() overwrote an existing approval") + } + stored, err := os.ReadFile(filepath.Join(kitDir, "approval.json")) + if err != nil || !bytes.Equal(stored, raw) { + t.Fatalf("stored approval mismatch: err=%v", err) + } +} + +func TestVerifyApprovedRejectsUnapprovedChannelAndManifestChange(t *testing.T) { + kitDir := buildWrittenKit(t) + approval, raw, err := Approve(kitDir, validApprovalRequest()) + if err != nil { + t.Fatal(err) + } + if err := WriteApproval(kitDir, raw); err != nil { + t.Fatal(err) + } + if _, _, err := VerifyApproved(kitDir, "instagram"); err == nil || !strings.Contains(err.Error(), "not approved") { + t.Fatalf("VerifyApproved() unapproved-channel error = %v", err) + } + + manifestPath := filepath.Join(kitDir, "manifest.json") + manifestRaw, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatal(err) + } + var manifest Manifest + if err := json.Unmarshal(manifestRaw, &manifest); err != nil { + t.Fatal(err) + } + manifest.PrimaryGoal = "modified after approval" + changed, _ := json.MarshalIndent(manifest, "", " ") + if err := os.WriteFile(manifestPath, append(changed, '\n'), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := VerifyApproved(kitDir, approval.Channels[0]); err == nil || !strings.Contains(err.Error(), "digest does not match") { + t.Fatalf("VerifyApproved() changed-manifest error = %v", err) + } +} + +func TestApproveRejectsTamperedMissingAndUntrackedFiles(t *testing.T) { + t.Run("tampered", func(t *testing.T) { + kitDir := buildWrittenKit(t) + path := filepath.Join(kitDir, "email", "body.html") + if err := os.WriteFile(path, []byte("changed"), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := Approve(kitDir, validApprovalRequest()); err == nil || !strings.Contains(err.Error(), "size changed") { + t.Fatalf("Approve() tamper error = %v", err) + } + }) + + t.Run("caption-tampered", func(t *testing.T) { + kitDir := buildWrittenKit(t) + path := filepath.Join(kitDir, "carousel", "caption.md") + if err := os.WriteFile(path, []byte("uncited replacement caption\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := Approve(kitDir, validApprovalRequest()); err == nil || (!strings.Contains(err.Error(), "size changed") && !strings.Contains(err.Error(), "digest changed")) { + t.Fatalf("Approve() caption tamper error = %v", err) + } + }) + + t.Run("missing", func(t *testing.T) { + kitDir := buildWrittenKit(t) + if err := os.Remove(filepath.Join(kitDir, "carousel", "slide-06.png")); err != nil { + t.Fatal(err) + } + if _, _, err := Approve(kitDir, validApprovalRequest()); err == nil || !strings.Contains(err.Error(), "missing files") { + t.Fatalf("Approve() missing-file error = %v", err) + } + }) + + t.Run("untracked", func(t *testing.T) { + kitDir := buildWrittenKit(t) + if err := os.WriteFile(filepath.Join(kitDir, "recipient-list.csv"), []byte("must not be stored here"), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := Approve(kitDir, validApprovalRequest()); err == nil || !strings.Contains(err.Error(), "untracked file") { + t.Fatalf("Approve() untracked-file error = %v", err) + } + }) +} + +func TestApproveRejectsMissingAttestationAndUnsafeManifest(t *testing.T) { + kitDir := buildWrittenKit(t) + request := validApprovalRequest() + request.Attested = false + if _, _, err := Approve(kitDir, request); err == nil || !strings.Contains(err.Error(), "attestation") { + t.Fatalf("Approve() attestation error = %v", err) + } + request = validApprovalRequest() + request.Audience = "reader@example.invalid" + if _, _, err := Approve(kitDir, request); err == nil || !strings.Contains(err.Error(), "non-sensitive label") { + t.Fatalf("Approve() sensitive-audience error = %v", err) + } + + manifestPath := filepath.Join(kitDir, "manifest.json") + raw, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatal(err) + } + var manifest Manifest + if err := json.Unmarshal(raw, &manifest); err != nil { + t.Fatal(err) + } + manifest.SendAuthorized = true + unsafeRaw, _ := marshalCanonicalJSON(manifest) + if err := os.WriteFile(manifestPath, unsafeRaw, 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := Approve(kitDir, validApprovalRequest()); err == nil || !strings.Contains(err.Error(), "review-pending") { + t.Fatalf("Approve() unsafe-manifest error = %v", err) + } +} + +func buildWrittenKit(t *testing.T) string { + t.Helper() + bundle, err := Build(readSampleBriefing(t), "https://example.org/TradeGravity/") + if err != nil { + t.Fatal(err) + } + kitDir := filepath.Join(t.TempDir(), "kit") + if err := Write(kitDir, bundle); err != nil { + t.Fatal(err) + } + return kitDir +} + +func validApprovalRequest() ApprovalRequest { + return ApprovalRequest{ + Reviewer: "reviewer", + Audience: "internal-pilot", + Channels: []string{"email"}, + ApprovedAt: time.Date(2026, 7, 17, 0, 0, 0, 0, time.UTC), + Attested: true, + } +} diff --git a/internal/distributionkit/kit.go b/internal/distributionkit/kit.go new file mode 100644 index 0000000..06fa48e --- /dev/null +++ b/internal/distributionkit/kit.go @@ -0,0 +1,617 @@ +package distributionkit + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "html" + "io/fs" + "net" + "net/url" + "os" + "path/filepath" + "sort" + "strings" +) + +const ( + kitSchemaVersion = "1.0" + kitToolVersion = "tradegravity-distributor/1.1" + captionRuneLimit = 1800 +) + +var readySignalKinds = []string{"reporter_total_change", "anchor_share_shift", "product_total_change"} +var readySlideRoles = []string{"cover", "scale", "anchor_balance", "product", "method", "cta"} + +type briefing struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt string `json:"generated_at"` + EditionID string `json:"edition_id"` + Status string `json:"status"` + Title string `json:"title"` + Scope string `json:"scope"` + LatestPeriod string `json:"latest_period"` + PreviousPeriod string `json:"previous_period"` + ReviewRequired bool `json:"review_required"` + Signals []signal `json:"signals"` + Email emailDraft `json:"email"` + SocialCarousel carousel `json:"social_carousel"` + Caveats []string `json:"caveats"` + EvidenceEntryPoint string `json:"evidence_entry_point"` +} + +type signal struct { + ID string `json:"id"` + Kind string `json:"kind"` + Title string `json:"title"` + Summary string `json:"summary"` + Period string `json:"period"` + PreviousPeriod string `json:"previous_period"` + Evidence []string `json:"evidence"` +} + +type emailDraft struct { + Subject string `json:"subject"` + Preview string `json:"preview"` + Markdown string `json:"markdown"` + CTALabel string `json:"cta_label"` + CTAPath string `json:"cta_path"` + SendPolicy string `json:"send_policy"` + PrimaryGoal string `json:"primary_goal"` +} + +type carousel struct { + Format string `json:"format"` + AspectRatio string `json:"aspect_ratio"` + ReviewPolicy string `json:"review_policy"` + Slides []slide `json:"slides"` +} + +type slide struct { + Order int `json:"order"` + Role string `json:"role"` + Headline string `json:"headline"` + Body string `json:"body"` + Evidence []string `json:"evidence"` +} + +type Manifest struct { + SchemaVersion string `json:"schema_version"` + Tool string `json:"tool"` + EditionID string `json:"edition_id"` + SourceGeneratedAt string `json:"source_generated_at"` + DistributionStatus string `json:"distribution_status"` + ReviewRequired bool `json:"review_required"` + SendAuthorized bool `json:"send_authorized"` + SocialPublishAuthorized bool `json:"social_publish_authorized"` + BaseURL string `json:"base_url"` + PrimaryGoal string `json:"primary_goal"` + Email ManifestEmail `json:"email"` + Carousel ManifestSocial `json:"carousel"` + Files []ManifestFile `json:"files"` +} + +type ManifestEmail struct { + Subject string `json:"subject"` + Preview string `json:"preview"` + CTAURL string `json:"cta_url"` +} + +type ManifestSocial struct { + Theme string `json:"theme"` + CaptionPath string `json:"caption_path"` + AspectRatio string `json:"aspect_ratio"` + Width int `json:"width"` + Height int `json:"height"` + SlideCount int `json:"slide_count"` + Formats []string `json:"formats"` +} + +type ManifestFile struct { + Path string `json:"path"` + MediaType string `json:"media_type"` + Bytes int `json:"bytes"` + SHA256 string `json:"sha256"` +} + +type Bundle struct { + Manifest Manifest + Files map[string][]byte +} + +func Build(raw []byte, baseURL string) (Bundle, error) { + return BuildWithOptions(raw, baseURL, BuildOptions{}) +} + +func BuildWithOptions(raw []byte, baseURL string, options BuildOptions) (Bundle, error) { + theme, err := resolveTheme(options.Theme) + if err != nil { + return Bundle{}, err + } + var source briefing + if err := json.Unmarshal(raw, &source); err != nil { + return Bundle{}, fmt.Errorf("decode briefing: %w", err) + } + if err := validateBriefing(source); err != nil { + return Bundle{}, err + } + root, err := normalizeBaseURL(baseURL) + if err != nil { + return Bundle{}, err + } + + ctaURL, err := resolveRootReference(root, source.Email.CTAPath) + if err != nil { + return Bundle{}, fmt.Errorf("resolve email CTA: %w", err) + } + files := map[string][]byte{ + "email/subject.txt": []byte(source.Email.Subject + "\n"), + "email/preview.txt": []byte(source.Email.Preview + "\n"), + "email/body.md": renderEmailMarkdown(source, root), + "email/body.html": renderEmailHTML(source, ctaURL), + "REVIEW.md": renderReviewChecklist(source, root), + } + fonts, err := newSlideFonts() + if err != nil { + return Bundle{}, err + } + defer fonts.Close() + + for _, item := range source.SocialCarousel.Slides { + svgPath := fmt.Sprintf("carousel/slide-%02d.svg", item.Order) + files[svgPath] = renderSlideSVG(source, item, root, theme) + pngPath := fmt.Sprintf("carousel/slide-%02d.png", item.Order) + pngContent, renderErr := renderSlidePNG(source, item, compactBaseURL(root), fonts, theme) + if renderErr != nil { + return Bundle{}, renderErr + } + files[pngPath] = pngContent + } + files["carousel/alt-text.md"] = renderAltText(source, root) + caption, err := renderInstagramCaption(source, root) + if err != nil { + return Bundle{}, err + } + files["carousel/caption.md"] = caption + files["carousel/index.html"] = renderCarouselIndex(source) + + manifest := Manifest{ + SchemaVersion: kitSchemaVersion, + Tool: kitToolVersion, + EditionID: source.EditionID, + SourceGeneratedAt: source.GeneratedAt, + DistributionStatus: "review_pending", + ReviewRequired: true, + SendAuthorized: false, + SocialPublishAuthorized: false, + BaseURL: root.String(), + PrimaryGoal: source.Email.PrimaryGoal, + Email: ManifestEmail{ + Subject: source.Email.Subject, + Preview: source.Email.Preview, + CTAURL: ctaURL, + }, + Carousel: ManifestSocial{ + Theme: theme.name, + CaptionPath: "carousel/caption.md", + AspectRatio: "4:5", + Width: 1080, + Height: 1350, + SlideCount: len(source.SocialCarousel.Slides), + Formats: []string{"png", "svg"}, + }, + } + paths := make([]string, 0, len(files)) + for path := range files { + paths = append(paths, path) + } + sort.Strings(paths) + for _, path := range paths { + content := files[path] + digest := sha256.Sum256(content) + manifest.Files = append(manifest.Files, ManifestFile{ + Path: path, + MediaType: mediaType(path), + Bytes: len(content), + SHA256: hex.EncodeToString(digest[:]), + }) + } + manifestJSON, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return Bundle{}, fmt.Errorf("encode manifest: %w", err) + } + files["manifest.json"] = append(manifestJSON, '\n') + return Bundle{Manifest: manifest, Files: files}, nil +} + +func Write(outputDir string, bundle Bundle) error { + outputDir = strings.TrimSpace(outputDir) + if outputDir == "" { + return errors.New("output directory is required") + } + abs, err := filepath.Abs(outputDir) + if err != nil { + return fmt.Errorf("resolve output directory: %w", err) + } + if _, err := os.Stat(abs); err == nil { + return fmt.Errorf("output directory already exists: %s", abs) + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect output directory: %w", err) + } + parent := filepath.Dir(abs) + if err := os.MkdirAll(parent, 0o755); err != nil { + return fmt.Errorf("create output parent: %w", err) + } + tempDir, err := os.MkdirTemp(parent, ".tradegravity-distribution-*") + if err != nil { + return fmt.Errorf("create temporary output: %w", err) + } + defer os.RemoveAll(tempDir) + + paths := make([]string, 0, len(bundle.Files)) + for path := range bundle.Files { + paths = append(paths, path) + } + sort.Strings(paths) + for _, path := range paths { + if !fs.ValidPath(path) || path == "." { + return fmt.Errorf("invalid bundle path %q", path) + } + target := filepath.Join(tempDir, filepath.FromSlash(path)) + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return fmt.Errorf("create bundle directory for %s: %w", path, err) + } + if err := os.WriteFile(target, bundle.Files[path], 0o644); err != nil { + return fmt.Errorf("write bundle file %s: %w", path, err) + } + } + if err := os.Rename(tempDir, abs); err != nil { + return fmt.Errorf("publish distribution kit: %w", err) + } + return nil +} + +func validateBriefing(source briefing) error { + if source.SchemaVersion != "1.0" || source.Status != "ready" { + return errors.New("briefing must be a ready schema 1.0 artifact") + } + if strings.TrimSpace(source.GeneratedAt) == "" || !safeIdentifier(source.EditionID) { + return errors.New("briefing has invalid provenance") + } + if !source.ReviewRequired || source.Email.SendPolicy != "manual_review_required" || source.SocialCarousel.ReviewPolicy != "manual_review_required" { + return errors.New("briefing must require manual review for email and social output") + } + if source.SocialCarousel.Format != "carousel_copy" || source.SocialCarousel.AspectRatio != "4:5" { + return errors.New("briefing carousel must use the reviewed 4:5 copy contract") + } + if len(source.Signals) != len(readySignalKinds) || len(source.SocialCarousel.Slides) != len(readySlideRoles) { + return errors.New("briefing must contain three signals and six slides") + } + if strings.TrimSpace(source.Email.Subject) == "" || strings.TrimSpace(source.Email.Preview) == "" || strings.TrimSpace(source.Email.Markdown) == "" || strings.TrimSpace(source.Email.CTALabel) == "" || strings.TrimSpace(source.Email.PrimaryGoal) == "" { + return errors.New("briefing email contract is incomplete") + } + if !validRelativeHref(source.Email.CTAPath) || !validRelativeHref(source.EvidenceEntryPoint) { + return errors.New("briefing contains an invalid evidence entry point") + } + for index, item := range source.Signals { + if item.Kind != readySignalKinds[index] || strings.TrimSpace(item.ID) == "" || strings.TrimSpace(item.Title) == "" || strings.TrimSpace(item.Summary) == "" { + return fmt.Errorf("briefing signal %d does not match the ready contract", index+1) + } + if item.Period != source.LatestPeriod || item.PreviousPeriod != source.PreviousPeriod || len(item.Evidence) < 2 { + return fmt.Errorf("briefing signal %s does not share the edition period and evidence", item.ID) + } + for _, href := range item.Evidence { + if !validRelativeHref(href) { + return fmt.Errorf("briefing signal %s has invalid evidence", item.ID) + } + } + } + for index, item := range source.SocialCarousel.Slides { + if item.Order != index+1 || item.Role != readySlideRoles[index] || strings.TrimSpace(item.Headline) == "" || strings.TrimSpace(item.Body) == "" || len(item.Evidence) == 0 { + return fmt.Errorf("briefing slide %d does not match the ready contract", index+1) + } + for _, href := range item.Evidence { + if !validRelativeHref(href) { + return fmt.Errorf("briefing slide %d has invalid evidence", item.Order) + } + } + } + return nil +} + +func normalizeBaseURL(value string) (*url.URL, error) { + parsed, err := url.Parse(strings.TrimSpace(value)) + if err != nil || parsed.Host == "" { + return nil, errors.New("base URL must be an absolute HTTPS URL") + } + loopback := parsed.Scheme == "http" && isLoopbackHost(parsed.Hostname()) + if parsed.Scheme != "https" && !loopback { + return nil, errors.New("base URL must use HTTPS except for a loopback preview") + } + if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return nil, errors.New("base URL must not contain credentials, query parameters, or a fragment") + } + if !strings.HasSuffix(parsed.Path, "/") { + parsed.Path += "/" + } + return parsed, nil +} + +func isLoopbackHost(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +func safeIdentifier(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '-' || r == '_' || r == '.' { + continue + } + return false + } + return true +} + +func validRelativeHref(value string) bool { + if !strings.HasPrefix(value, "./") || strings.Contains(value, "\\") || strings.ContainsAny(value, "\r\n") { + return false + } + parsed, err := url.Parse(value) + if err != nil || parsed.IsAbs() || parsed.Host != "" { + return false + } + for _, part := range strings.Split(parsed.Path, "/") { + if part == ".." { + return false + } + } + return true +} + +func resolveRootReference(root *url.URL, href string) (string, error) { + if !validRelativeHref(href) { + return "", errors.New("reference must be a safe same-origin relative path") + } + ref, err := url.Parse(href) + if err != nil { + return "", err + } + return root.ResolveReference(ref).String(), nil +} + +func resolveEvidenceReference(root *url.URL, href string) string { + if !validRelativeHref(href) { + return "" + } + ref, _ := url.Parse(href) + if ref.RawQuery != "" && (ref.Path == "." || ref.Path == "./" || ref.Path == "") { + return root.ResolveReference(ref).String() + } + dataRef, _ := url.Parse("data/") + dataRoot := root.ResolveReference(dataRef) + return dataRoot.ResolveReference(ref).String() +} + +func materializeMarkdown(markdown string, root *url.URL) string { + base := strings.TrimSuffix(root.String(), "/") + return strings.ReplaceAll(markdown, "{{BASE_URL}}", base) +} + +func renderEmailHTML(source briefing, ctaURL string) []byte { + var body strings.Builder + body.WriteString("\n") + body.WriteString("" + html.EscapeString(source.Email.Subject) + "") + body.WriteString("") + body.WriteString("
" + html.EscapeString(source.Email.Preview) + "
") + body.WriteString("
") + body.WriteString("") + body.WriteString("") + for _, item := range source.Signals { + body.WriteString("") + } + body.WriteString("") + body.WriteString("") + body.WriteString("
TRADEGRAVITY · REVIEWED DRAFT
") + body.WriteString("

" + html.EscapeString(source.Title) + "

") + body.WriteString("

Observation window: " + html.EscapeString(source.LatestPeriod) + " vs " + html.EscapeString(source.PreviousPeriod) + "

") + body.WriteString("

" + html.EscapeString(item.Title) + "

") + body.WriteString("

" + html.EscapeString(item.Summary) + "

") + body.WriteString("

" + html.EscapeString(item.PreviousPeriod) + " → " + html.EscapeString(item.Period) + " · cited evidence retained in the kit manifest

" + html.EscapeString(source.Email.CTALabel) + "

" + html.EscapeString(source.Scope) + "

No tracking pixel is included. Sending remains unauthorized until the review checklist and subscriber-consent controls are completed.

Unsubscribe

\n") + return []byte(body.String()) +} + +func renderEmailMarkdown(source briefing, root *url.URL) []byte { + body := strings.TrimSpace(materializeMarkdown(source.Email.Markdown, root)) + body += "\n\n---\n\nUnsubscribe: {{UNSUBSCRIBE_URL}}\n" + return []byte(body) +} + +func renderSlideSVG(source briefing, item slide, root *url.URL, theme cardTheme) []byte { + headlineLines := wrapText(item.Headline, 28, 4) + bodyLines := wrapText(item.Body, 45, 6) + evidenceLines := make([]string, 0, 3) + for _, href := range item.Evidence { + if len(evidenceLines) == 2 { + break + } + evidenceLines = append(evidenceLines, truncateRunes(cardEvidenceLabel(href), 72)) + } + evidenceLines = append(evidenceLines, truncateRunes("OPEN · "+compactBaseURL(root), 72)) + accent := theme.accent(item.Role) + role := strings.ToUpper(strings.ReplaceAll(item.Role, "_", " ")) + + var svg strings.Builder + svg.WriteString("\n") + svg.WriteString("" + html.EscapeString(item.Headline) + "" + html.EscapeString(item.Body) + "\n") + svg.WriteString("\n") + svg.WriteString("\n") + svg.WriteString(fmt.Sprintf("\n", theme.frame, float64(theme.frameAlpha)/255)) + svg.WriteString("TRADEGRAVITY · US–CHINA CHIP LENS\n") + svg.WriteString(fmt.Sprintf("%02d / %02d\n", theme.muted, item.Order, len(source.SocialCarousel.Slides))) + svg.WriteString("") + svg.WriteString("" + html.EscapeString(role) + "\n") + writeSVGLines(&svg, headlineLines, 88, 330, 64, 78, "700", theme.headline) + writeSVGLines(&svg, bodyLines, 88, 700, 34, 53, "400", theme.body) + svg.WriteString(fmt.Sprintf("\n", theme.panel, float64(theme.panelAlpha)/255, theme.panelBorder, float64(theme.panelBorderAlpha)/255)) + svg.WriteString("EVIDENCE\n") + writeSVGLines(&svg, evidenceLines, 116, 1144, 20, 29, "400", theme.muted) + svg.WriteString("Reviewed draft · descriptive customs evidence · not investment advice\n") + svg.WriteString("\n") + return []byte(svg.String()) +} + +func renderAltText(source briefing, root *url.URL) []byte { + var result strings.Builder + result.WriteString("# Carousel alt text\n\n") + result.WriteString("Review and edit this text with each final visual before publishing.\n\n") + for _, item := range source.SocialCarousel.Slides { + result.WriteString(fmt.Sprintf("## Slide %d — %s\n\n", item.Order, item.Headline)) + result.WriteString(item.Headline + ". " + item.Body + "\n\n") + result.WriteString("Evidence:\n") + for _, href := range item.Evidence { + result.WriteString("- " + resolveEvidenceReference(root, href) + "\n") + } + result.WriteString("\n") + } + return []byte(result.String()) +} + +func renderInstagramCaption(source briefing, root *url.URL) ([]byte, error) { + plain := func(value string) string { return strings.Join(strings.Fields(value), " ") } + var caption strings.Builder + caption.WriteString(plain(source.Title) + "\n") + caption.WriteString(plain(source.LatestPeriod) + " vs " + plain(source.PreviousPeriod) + "\n\n") + for _, item := range source.Signals { + caption.WriteString("• " + plain(item.Title) + " — " + plain(item.Summary) + "\n") + } + caption.WriteString("\nExplore the cited evidence and methodology:\n") + caption.WriteString(resolveEvidenceReference(root, source.EvidenceEntryPoint) + "\n\n") + caption.WriteString("Scope note: descriptive customs evidence; not a physical shipment route, causal claim, or investment recommendation.\n\n") + caption.WriteString("#TradeGravity #Semiconductors #SupplyChain #USChinaTrade\n") + result := caption.String() + if len([]rune(result)) > captionRuneLimit { + return nil, fmt.Errorf("Instagram caption draft exceeds the %d-rune editorial ceiling", captionRuneLimit) + } + return []byte(result), nil +} + +func renderCarouselIndex(source briefing) []byte { + var page strings.Builder + page.WriteString("" + html.EscapeString(source.EditionID) + " carousel review") + page.WriteString("
") + page.WriteString("

Review pending · this page does not publish or send anything.

" + html.EscapeString(source.Title) + "

Open the Instagram caption draft · Open alt text

") + for _, item := range source.SocialCarousel.Slides { + page.WriteString(fmt.Sprintf("
\"%s\"
Slide %d · %s · PNG upload asset
", item.Order, html.EscapeString(item.Headline+". "+item.Body), item.Order, html.EscapeString(item.Role))) + } + page.WriteString("
\n") + return []byte(page.String()) +} + +func renderReviewChecklist(source briefing, root *url.URL) []byte { + var review strings.Builder + review.WriteString("# Distribution review — " + source.EditionID + "\n\n") + review.WriteString("Status: **review pending**. Generating this kit does not authorize email delivery or social publication.\n\n") + review.WriteString("- Source publication: `" + source.GeneratedAt + "`\n") + review.WriteString("- Observation window: `" + source.PreviousPeriod + "` → `" + source.LatestPeriod + "`\n") + review.WriteString("- Evidence entry point: " + resolveEvidenceReference(root, source.EvidenceEntryPoint) + "\n\n") + review.WriteString("## Editorial and evidence\n\n- [ ] Periods, values, direction, and units match the cited JSON.\n- [ ] Every evidence URL opens and the source scope is still accurate.\n- [ ] Monthly movement is not described as publication revision, causality, capacity, or a physical route.\n- [ ] Subject, preview, one primary CTA, and mobile rendering were reviewed.\n- [ ] Carousel text and supplied alt text remain legible at feed size.\n- [ ] `carousel/caption.md`, tags, evidence link, and scope note were reviewed together with the final cards.\n- [ ] A named editor approved the final copy and exported images.\n\n") + review.WriteString("## Delivery and privacy\n\n- [ ] Recipients completed double opt-in for this publication.\n- [ ] The `{{UNSUBSCRIBE_URL}}` placeholder, one-click unsubscribe headers, and suppression handling were tested.\n- [ ] Sender identity, SPF, DKIM, and DMARC are configured.\n- [ ] Bounce, complaint, retention, and deletion procedures are documented.\n- [ ] No subscriber addresses, provider credentials, or tracking secrets are stored in this kit.\n\n") + review.WriteString("## Approval\n\n- Editor: ____________________\n- Date: ______________________\n- Approved channels: ____________________\n- Non-sensitive audience label: ____________________\n- Final asset hashes recorded: [ ]\n\nAfter every box above is complete, create the content-release record with an explicit UTC time:\n\n```bash\ngo run ./cmd/distribution-approval \\\n -kit distribution-kit \\\n -reviewer YOUR_HANDLE \\\n -audience consented-internal-pilot \\\n -channels email,instagram \\\n -approved-at 2026-07-17T12:00:00Z \\\n -attest-reviewed\n```\n\nThis records content approval only. It does not certify subscriber consent, provider readiness, or automatic publishing permission.\n") + return []byte(review.String()) +} + +func wrapText(value string, maxRunes, maxLines int) []string { + words := strings.Fields(value) + if len(words) == 0 { + return []string{""} + } + lines := make([]string, 0, maxLines) + current := "" + for _, word := range words { + candidate := word + if current != "" { + candidate = current + " " + word + } + if len([]rune(candidate)) <= maxRunes { + current = candidate + continue + } + if current != "" { + lines = append(lines, current) + } + current = truncateRunes(word, maxRunes) + } + if current != "" { + lines = append(lines, current) + } + if len(lines) > maxLines { + lines = lines[:maxLines] + lines[maxLines-1] = truncateRunes(strings.TrimSpace(lines[maxLines-1])+" …", maxRunes) + } + return lines +} + +func truncateRunes(value string, limit int) string { + runes := []rune(value) + if len(runes) <= limit { + return value + } + if limit <= 1 { + return "…" + } + return string(runes[:limit-1]) + "…" +} + +func writeSVGLines(output *strings.Builder, lines []string, x, y, size, lineHeight int, weight, fill string) { + for index, line := range lines { + output.WriteString(fmt.Sprintf("%s\n", x, y+index*lineHeight, fill, size, weight, html.EscapeString(line))) + } +} + +func cardEvidenceLabel(href string) string { + parsed, err := url.Parse(href) + if err != nil { + return "EVIDENCE · review manifest" + } + if parsed.RawQuery != "" && (parsed.Path == "." || parsed.Path == "./" || parsed.Path == "") { + return "APP · ?" + parsed.RawQuery + } + path := strings.TrimPrefix(parsed.Path, "./") + return "DATA · " + path +} + +func compactBaseURL(root *url.URL) string { + return root.Host + strings.TrimSuffix(root.EscapedPath(), "/") +} + +func mediaType(path string) string { + switch filepath.Ext(path) { + case ".html": + return "text/html; charset=utf-8" + case ".json": + return "application/json" + case ".md": + return "text/markdown; charset=utf-8" + case ".svg": + return "image/svg+xml" + case ".png": + return "image/png" + default: + return "text/plain; charset=utf-8" + } +} + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/internal/distributionkit/kit_test.go b/internal/distributionkit/kit_test.go new file mode 100644 index 0000000..acfdec7 --- /dev/null +++ b/internal/distributionkit/kit_test.go @@ -0,0 +1,202 @@ +package distributionkit + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "fmt" + "image/png" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestBuildProducesDeterministicReviewGatedKit(t *testing.T) { + raw := readSampleBriefing(t) + first, err := Build(raw, "https://example.org/TradeGravity/") + if err != nil { + t.Fatalf("Build() error = %v", err) + } + second, err := Build(raw, "https://example.org/TradeGravity/") + if err != nil { + t.Fatalf("second Build() error = %v", err) + } + if !bytes.Equal(first.Files["manifest.json"], second.Files["manifest.json"]) { + t.Fatal("manifest is not deterministic") + } + if !bytes.Equal(first.Files["carousel/slide-01.png"], second.Files["carousel/slide-01.png"]) { + t.Fatal("PNG rendering is not deterministic") + } + if first.Manifest.DistributionStatus != "review_pending" || first.Manifest.SendAuthorized || first.Manifest.SocialPublishAuthorized || !first.Manifest.ReviewRequired { + t.Fatalf("unsafe manifest gates: %+v", first.Manifest) + } + if first.Manifest.Carousel.Width != 1080 || first.Manifest.Carousel.Height != 1350 || first.Manifest.Carousel.SlideCount != 6 || strings.Join(first.Manifest.Carousel.Formats, ",") != "png,svg" { + t.Fatalf("unexpected social dimensions: %+v", first.Manifest.Carousel) + } + if first.Manifest.Carousel.Theme != ThemeIntelligenceDark { + t.Fatalf("default carousel theme = %q", first.Manifest.Carousel.Theme) + } + if len(first.Files) != 21 || len(first.Manifest.Files) != 20 { + t.Fatalf("file counts = %d/%d, want 21/20", len(first.Files), len(first.Manifest.Files)) + } + if first.Manifest.Carousel.CaptionPath != "carousel/caption.md" { + t.Fatalf("caption path = %q", first.Manifest.Carousel.CaptionPath) + } + + htmlBody := string(first.Files["email/body.html"]) + if strings.Count(htmlBody, " captionRuneLimit || strings.Contains(caption, "{{") { + t.Fatal("caption exceeded the editorial contract or retained a placeholder") + } +} + +func TestBuildEditorialThemeIsDeterministicDistinctAndReviewGated(t *testing.T) { + raw := readSampleBriefing(t) + options := BuildOptions{Theme: ThemeEditorialLight} + first, err := BuildWithOptions(raw, "https://example.org/TradeGravity/", options) + if err != nil { + t.Fatal(err) + } + second, err := BuildWithOptions(raw, "https://example.org/TradeGravity/", options) + if err != nil { + t.Fatal(err) + } + baseline, err := Build(raw, "https://example.org/TradeGravity/") + if err != nil { + t.Fatal(err) + } + if first.Manifest.Carousel.Theme != ThemeEditorialLight || first.Manifest.SocialPublishAuthorized || !first.Manifest.ReviewRequired { + t.Fatalf("unsafe or missing editorial theme manifest: %+v", first.Manifest) + } + if !bytes.Equal(first.Files["manifest.json"], second.Files["manifest.json"]) || !bytes.Equal(first.Files["carousel/slide-01.png"], second.Files["carousel/slide-01.png"]) { + t.Fatal("editorial theme output is not deterministic") + } + if bytes.Equal(first.Files["carousel/slide-01.png"], baseline.Files["carousel/slide-01.png"]) || bytes.Equal(first.Files["carousel/slide-01.svg"], baseline.Files["carousel/slide-01.svg"]) { + t.Fatal("editorial theme did not produce distinct review assets") + } + for index := 1; index <= 6; index++ { + path := fmt.Sprintf("carousel/slide-%02d.png", index) + decoded, err := png.Decode(bytes.NewReader(first.Files[path])) + if err != nil || decoded.Bounds().Dx() != 1080 || decoded.Bounds().Dy() != 1350 { + t.Fatalf("editorial asset %s is invalid: bounds=%v err=%v", path, decoded.Bounds(), err) + } + } + kitDir := filepath.Join(t.TempDir(), "editorial-kit") + if err := Write(kitDir, first); err != nil { + t.Fatal(err) + } + _, approvalRaw, err := Approve(kitDir, ApprovalRequest{Reviewer: "reviewer", Audience: "editorial-pilot", Channels: []string{"instagram"}, ApprovedAt: time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC), Attested: true}) + if err != nil { + t.Fatal(err) + } + if err := WriteApproval(kitDir, approvalRaw); err != nil { + t.Fatal(err) + } + if _, _, err := VerifyApproved(kitDir, "instagram"); err != nil { + t.Fatalf("editorial theme could not cross the existing approval seam: %v", err) + } +} + +func TestBuildRejectsUnknownTheme(t *testing.T) { + if _, err := BuildWithOptions(readSampleBriefing(t), "https://example.org/", BuildOptions{Theme: "remote-html"}); err == nil { + t.Fatal("BuildWithOptions() accepted an unknown renderer theme") + } +} + +func TestBuildRejectsAutomaticOrUnavailableBriefing(t *testing.T) { + raw := readSampleBriefing(t) + var value map[string]any + if err := json.Unmarshal(raw, &value); err != nil { + t.Fatal(err) + } + value["review_required"] = false + unsafe, _ := json.Marshal(value) + if _, err := Build(unsafe, "https://example.org/"); err == nil { + t.Fatal("Build() accepted a briefing without manual review") + } + value["review_required"] = true + value["status"] = "unavailable" + unavailable, _ := json.Marshal(value) + if _, err := Build(unavailable, "https://example.org/"); err == nil { + t.Fatal("Build() accepted an unavailable briefing") + } + if _, err := Build(raw, "http://example.org/"); err == nil { + t.Fatal("Build() accepted an insecure public base URL") + } +} + +func TestWriteCreatesNewDirectoryAndRefusesOverwrite(t *testing.T) { + bundle, err := Build(readSampleBriefing(t), "http://127.0.0.1:8080/") + if err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "kit") + if err := Write(out, bundle); err != nil { + t.Fatalf("Write() error = %v", err) + } + if _, err := os.Stat(filepath.Join(out, "carousel", "slide-01.png")); err != nil { + t.Fatalf("written kit is incomplete: %v", err) + } + if err := Write(out, bundle); err == nil { + t.Fatal("Write() overwrote an existing directory") + } +} + +func readSampleBriefing(t *testing.T) []byte { + t.Helper() + path := filepath.Join("..", "..", "examples", "sample-data", "briefing.json") + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read sample briefing: %v", err) + } + return raw +} diff --git a/internal/distributionkit/png.go b/internal/distributionkit/png.go new file mode 100644 index 0000000..546403b --- /dev/null +++ b/internal/distributionkit/png.go @@ -0,0 +1,313 @@ +package distributionkit + +import ( + "bytes" + "fmt" + "image" + "image/color" + "image/png" + "io" + "strings" + + "golang.org/x/image/font" + "golang.org/x/image/font/gofont/gobold" + "golang.org/x/image/font/gofont/goregular" + "golang.org/x/image/font/opentype" + "golang.org/x/image/math/fixed" +) + +const ( + cardWidth = 1080 + cardHeight = 1350 +) + +type slideFonts struct { + header font.Face + counter font.Face + role font.Face + headline font.Face + body font.Face + evidenceTitle font.Face + evidence font.Face + footer font.Face + closers []io.Closer +} + +func newSlideFonts() (slideFonts, error) { + regular, err := opentype.Parse(goregular.TTF) + if err != nil { + return slideFonts{}, fmt.Errorf("parse embedded regular font: %w", err) + } + bold, err := opentype.Parse(gobold.TTF) + if err != nil { + return slideFonts{}, fmt.Errorf("parse embedded bold font: %w", err) + } + + var result slideFonts + create := func(target *font.Face, source *opentype.Font, size float64) error { + face, faceErr := opentype.NewFace(source, &opentype.FaceOptions{ + Size: size, + DPI: 72, + Hinting: font.HintingFull, + }) + if faceErr != nil { + return faceErr + } + *target = face + if closer, ok := face.(io.Closer); ok { + result.closers = append(result.closers, closer) + } + return nil + } + faces := []struct { + target *font.Face + source *opentype.Font + size float64 + }{ + {&result.header, bold, 24}, + {&result.counter, regular, 22}, + {&result.role, bold, 20}, + {&result.headline, bold, 64}, + {&result.body, regular, 34}, + {&result.evidenceTitle, bold, 18}, + {&result.evidence, regular, 20}, + {&result.footer, regular, 18}, + } + for _, spec := range faces { + if err := create(spec.target, spec.source, spec.size); err != nil { + result.Close() + return slideFonts{}, fmt.Errorf("create embedded font face: %w", err) + } + } + return result, nil +} + +func (fonts slideFonts) Close() { + for _, closer := range fonts.closers { + _ = closer.Close() + } +} + +func renderSlidePNG(source briefing, item slide, rootURL string, fonts slideFonts, theme cardTheme) ([]byte, error) { + canvas := image.NewRGBA(image.Rect(0, 0, cardWidth, cardHeight)) + drawCardBackground(canvas, theme) + + accent := mustHexColor(theme.accent(item.Role), 255) + drawCircle(canvas, 940, 120, 230, withAlpha(accent, 26)) + drawCircle(canvas, 80, 1260, 250, mustHexColor(theme.decoration, 18)) + drawRoundedRect(canvas, 64, 62, 952, 1226, 28, color.RGBA{}, mustHexColor(theme.frame, theme.frameAlpha), 2) + + drawCardText(canvas, fonts.header, 88, 126, mustHexColor(theme.header, 255), "TRADEGRAVITY · US–CHINA CHIP LENS") + counter := fmt.Sprintf("%02d / %02d", item.Order, len(source.SocialCarousel.Slides)) + drawRightAlignedText(canvas, fonts.counter, 992, 126, mustHexColor(theme.muted, 255), counter) + + role := strings.ToUpper(strings.ReplaceAll(item.Role, "_", " ")) + pillWidth := maxInt(210, 40+len([]rune(role))*15) + drawRoundedRect(canvas, 88, 178, pillWidth, 48, 24, withAlpha(accent, 41), withAlpha(accent, 166), 2) + drawCardText(canvas, fonts.role, 112, 210, accent, role) + + headlineLines := wrapTextForFace(item.Headline, fonts.headline, 880, 4) + bodyLines := wrapTextForFace(item.Body, fonts.body, 880, 6) + drawCardLines(canvas, fonts.headline, headlineLines, 88, 330, 78, mustHexColor(theme.headline, 255)) + drawCardLines(canvas, fonts.body, bodyLines, 88, 700, 53, mustHexColor(theme.body, 255)) + + drawRoundedRect(canvas, 88, 1060, 904, 154, 18, mustHexColor(theme.panel, theme.panelAlpha), mustHexColor(theme.panelBorder, theme.panelBorderAlpha), 2) + drawCardText(canvas, fonts.evidenceTitle, 116, 1102, mustHexColor(theme.evidenceTitle, 255), "EVIDENCE") + evidenceLines := make([]string, 0, 3) + for _, href := range item.Evidence { + if len(evidenceLines) == 2 { + break + } + evidenceLines = append(evidenceLines, truncateToWidth(cardEvidenceLabel(href), fonts.evidence, 840)) + } + evidenceLines = append(evidenceLines, truncateToWidth("OPEN · "+rootURL, fonts.evidence, 840)) + drawCardLines(canvas, fonts.evidence, evidenceLines, 116, 1144, 29, mustHexColor(theme.muted, 255)) + + drawCardText(canvas, fonts.footer, 88, 1260, mustHexColor(theme.footer, 255), "Reviewed draft · descriptive customs evidence · not investment advice") + + var output bytes.Buffer + encoder := png.Encoder{CompressionLevel: png.BestCompression} + if err := encoder.Encode(&output, canvas); err != nil { + return nil, fmt.Errorf("encode slide PNG: %w", err) + } + return output.Bytes(), nil +} + +func drawCardBackground(target *image.RGBA, theme cardTheme) { + start := mustHexColor(theme.backgroundStart, 255) + middle := mustHexColor(theme.backgroundMiddle, 255) + end := mustHexColor(theme.backgroundEnd, 255) + for y := 0; y < cardHeight; y++ { + for x := 0; x < cardWidth; x++ { + progress := float64(x+y) / float64(cardWidth+cardHeight-2) + if progress <= 0.56 { + target.SetRGBA(x, y, interpolateColor(start, middle, progress/0.56)) + continue + } + target.SetRGBA(x, y, interpolateColor(middle, end, (progress-0.56)/0.44)) + } + } +} + +func interpolateColor(from, to color.RGBA, progress float64) color.RGBA { + channel := func(a, b uint8) uint8 { + return uint8(float64(a) + (float64(b)-float64(a))*progress) + } + return color.RGBA{R: channel(from.R, to.R), G: channel(from.G, to.G), B: channel(from.B, to.B), A: 255} +} + +func drawCircle(target *image.RGBA, centerX, centerY, radius int, fill color.RGBA) { + for y := maxInt(0, centerY-radius); y < minInt(cardHeight, centerY+radius+1); y++ { + for x := maxInt(0, centerX-radius); x < minInt(cardWidth, centerX+radius+1); x++ { + dx, dy := x-centerX, y-centerY + if dx*dx+dy*dy <= radius*radius { + blendPixel(target, x, y, fill) + } + } + } +} + +func drawRoundedRect(target *image.RGBA, x, y, width, height, radius int, fill, stroke color.RGBA, strokeWidth int) { + for py := y; py < y+height; py++ { + for px := x; px < x+width; px++ { + if !insideRoundedRect(px, py, x, y, width, height, radius) { + continue + } + paint := stroke + inner := strokeWidth > 0 && insideRoundedRect(px, py, x+strokeWidth, y+strokeWidth, width-strokeWidth*2, height-strokeWidth*2, maxInt(0, radius-strokeWidth)) + if strokeWidth == 0 || inner { + paint = fill + } + blendPixel(target, px, py, paint) + } + } +} + +func insideRoundedRect(px, py, x, y, width, height, radius int) bool { + if width <= 0 || height <= 0 || px < x || px >= x+width || py < y || py >= y+height { + return false + } + if radius <= 0 || (px >= x+radius && px < x+width-radius) || (py >= y+radius && py < y+height-radius) { + return true + } + cx := x + radius + if px >= x+width-radius { + cx = x + width - radius - 1 + } + cy := y + radius + if py >= y+height-radius { + cy = y + height - radius - 1 + } + dx, dy := px-cx, py-cy + return dx*dx+dy*dy <= radius*radius +} + +func blendPixel(target *image.RGBA, x, y int, source color.RGBA) { + if source.A == 0 { + return + } + if source.A == 255 { + target.SetRGBA(x, y, source) + return + } + destination := target.RGBAAt(x, y) + alpha := uint32(source.A) + inverse := uint32(255 - source.A) + target.SetRGBA(x, y, color.RGBA{ + R: uint8((uint32(source.R)*alpha + uint32(destination.R)*inverse) / 255), + G: uint8((uint32(source.G)*alpha + uint32(destination.G)*inverse) / 255), + B: uint8((uint32(source.B)*alpha + uint32(destination.B)*inverse) / 255), + A: 255, + }) +} + +func drawCardText(target *image.RGBA, face font.Face, x, baseline int, fill color.RGBA, value string) { + drawer := font.Drawer{ + Dst: target, + Src: image.NewUniform(fill), + Face: face, + Dot: fixed.P(x, baseline), + } + drawer.DrawString(value) +} + +func drawRightAlignedText(target *image.RGBA, face font.Face, right, baseline int, fill color.RGBA, value string) { + width := font.MeasureString(face, value).Ceil() + drawCardText(target, face, right-width, baseline, fill, value) +} + +func drawCardLines(target *image.RGBA, face font.Face, lines []string, x, baseline, lineHeight int, fill color.RGBA) { + for index, line := range lines { + drawCardText(target, face, x, baseline+index*lineHeight, fill, line) + } +} + +func wrapTextForFace(value string, face font.Face, maxWidth, maxLines int) []string { + words := strings.Fields(value) + if len(words) == 0 { + return []string{""} + } + lines := make([]string, 0, maxLines) + current := "" + for _, word := range words { + candidate := word + if current != "" { + candidate = current + " " + word + } + if font.MeasureString(face, candidate).Ceil() <= maxWidth { + current = candidate + continue + } + if current != "" { + lines = append(lines, current) + } + current = truncateToWidth(word, face, maxWidth) + } + if current != "" { + lines = append(lines, current) + } + if len(lines) > maxLines { + lines = lines[:maxLines] + lines[maxLines-1] = truncateToWidth(strings.TrimSpace(lines[maxLines-1])+" …", face, maxWidth) + } + return lines +} + +func truncateToWidth(value string, face font.Face, maxWidth int) string { + if font.MeasureString(face, value).Ceil() <= maxWidth { + return value + } + runes := []rune(value) + for len(runes) > 0 { + candidate := strings.TrimSpace(string(runes)) + "…" + if font.MeasureString(face, candidate).Ceil() <= maxWidth { + return candidate + } + runes = runes[:len(runes)-1] + } + return "…" +} + +func mustHexColor(value string, alpha uint8) color.RGBA { + value = strings.TrimPrefix(value, "#") + if len(value) != 6 { + panic("invalid card color") + } + var red, green, blue uint8 + if _, err := fmt.Sscanf(value, "%02x%02x%02x", &red, &green, &blue); err != nil { + panic("invalid card color") + } + return color.RGBA{R: red, G: green, B: blue, A: alpha} +} + +func withAlpha(value color.RGBA, alpha uint8) color.RGBA { + value.A = alpha + return value +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/internal/distributionkit/theme.go b/internal/distributionkit/theme.go new file mode 100644 index 0000000..33d6864 --- /dev/null +++ b/internal/distributionkit/theme.go @@ -0,0 +1,58 @@ +package distributionkit + +import ( + "errors" + "strings" +) + +const ( + ThemeIntelligenceDark = "intelligence-dark" + ThemeEditorialLight = "editorial-light" +) + +type BuildOptions struct { + Theme string +} + +type cardTheme struct { + name string + backgroundStart, backgroundMiddle, backgroundEnd string + decoration, frame, header, muted, headline, body string + panel, panelBorder, evidenceTitle, footer string + frameAlpha, panelAlpha, panelBorderAlpha uint8 + accents map[string]string +} + +func resolveTheme(name string) (cardTheme, error) { + name = strings.TrimSpace(name) + if name == "" { + name = ThemeIntelligenceDark + } + switch name { + case ThemeIntelligenceDark: + return cardTheme{ + name: name, backgroundStart: "#111827", backgroundMiddle: "#0b0d12", backgroundEnd: "#161117", + decoration: "#5aa2ff", frame: "#ffffff", header: "#e7d37c", muted: "#9aa4b3", + headline: "#ffffff", body: "#c7cdd7", panel: "#ffffff", panelBorder: "#ffffff", + evidenceTitle: "#e7d37c", footer: "#7f8998", frameAlpha: 33, panelAlpha: 9, panelBorderAlpha: 26, + accents: map[string]string{"scale": "#5aa2ff", "anchor_balance": "#86e7b0", "product": "#ff8a68", "cta": "#74b3ff", "default": "#e7d37c"}, + }, nil + case ThemeEditorialLight: + return cardTheme{ + name: name, backgroundStart: "#f7f3ea", backgroundMiddle: "#ffffff", backgroundEnd: "#eee8dc", + decoration: "#557da6", frame: "#26323f", header: "#895f25", muted: "#5b6571", + headline: "#17202a", body: "#44505c", panel: "#ffffff", panelBorder: "#26323f", + evidenceTitle: "#895f25", footer: "#5b6571", frameAlpha: 46, panelAlpha: 235, panelBorderAlpha: 31, + accents: map[string]string{"scale": "#356b99", "anchor_balance": "#28745b", "product": "#a34d36", "cta": "#315f86", "default": "#895f25"}, + }, nil + default: + return cardTheme{}, errors.New("carousel theme must be intelligence-dark or editorial-light") + } +} + +func (theme cardTheme) accent(role string) string { + if value := theme.accents[role]; value != "" { + return value + } + return theme.accents["default"] +} diff --git a/internal/distributionkit/theme_test.go b/internal/distributionkit/theme_test.go new file mode 100644 index 0000000..b4175df --- /dev/null +++ b/internal/distributionkit/theme_test.go @@ -0,0 +1,73 @@ +package distributionkit + +import ( + "fmt" + "math" + "testing" +) + +func TestCardThemesMeetTextContrastContract(t *testing.T) { + for _, name := range []string{ThemeIntelligenceDark, ThemeEditorialLight} { + t.Run(name, func(t *testing.T) { + theme, err := resolveTheme(name) + if err != nil { + t.Fatal(err) + } + backgrounds := []string{theme.backgroundStart, theme.backgroundMiddle, theme.backgroundEnd} + foregrounds := map[string]string{ + "header": theme.header, "counter/evidence": theme.muted, "headline": theme.headline, + "body": theme.body, "evidence title": theme.evidenceTitle, "footer": theme.footer, + } + for role, foreground := range foregrounds { + for _, background := range backgrounds { + if ratio := contrastRatio(foreground, background); ratio < 4.5 { + t.Errorf("%s on %s contrast %.2f:1, want at least 4.5:1", role, background, ratio) + } + } + } + for role, foreground := range theme.accents { + for _, background := range backgrounds { + pill := blendHex(foreground, background, 0.16) + if ratio := contrastRatio(foreground, pill); ratio < 3.0 { + t.Errorf("role label %s on blended pill %s contrast %.2f:1, want at least 3:1", role, pill, ratio) + } + } + } + }) + } +} + +func contrastRatio(first, second string) float64 { + a, b := relativeLuminance(parseTestHex(first)), relativeLuminance(parseTestHex(second)) + if a < b { + a, b = b, a + } + return (a + 0.05) / (b + 0.05) +} + +func relativeLuminance(value [3]float64) float64 { + linear := func(channel float64) float64 { + channel /= 255 + if channel <= 0.04045 { + return channel / 12.92 + } + return math.Pow((channel+0.055)/1.055, 2.4) + } + return 0.2126*linear(value[0]) + 0.7152*linear(value[1]) + 0.0722*linear(value[2]) +} + +func blendHex(foreground, background string, alpha float64) string { + front, back := parseTestHex(foreground), parseTestHex(background) + return fmt.Sprintf("#%02x%02x%02x", + int(math.Round(front[0]*alpha+back[0]*(1-alpha))), + int(math.Round(front[1]*alpha+back[1]*(1-alpha))), + int(math.Round(front[2]*alpha+back[2]*(1-alpha)))) +} + +func parseTestHex(value string) [3]float64 { + var red, green, blue uint8 + if _, err := fmt.Sscanf(value, "#%02x%02x%02x", &red, &green, &blue); err != nil { + panic(err) + } + return [3]float64{float64(red), float64(green), float64(blue)} +} diff --git a/internal/emaildelivery/authorization.go b/internal/emaildelivery/authorization.go new file mode 100644 index 0000000..5e813c7 --- /dev/null +++ b/internal/emaildelivery/authorization.go @@ -0,0 +1,349 @@ +package emaildelivery + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/mail" + "os" + "path/filepath" + "strings" + "time" + + "tradegravity/internal/deliverypreflight" +) + +const ( + authorizationSchemaVersion = "1.0" + authorizationToolVersion = "tradegravity-email-launch-approval/1.0" + maximumAuthorizationAge = time.Hour + maximumPrivateFileSize = 5 << 20 +) + +// Attestations records the operator controls that cannot be inferred from the +// repository or the provider API. +type Attestations struct { + SenderDomainAuthenticated bool `json:"sender_domain_authenticated"` + BounceComplaintReady bool `json:"bounce_complaint_process_ready"` + PrivacyControlsReviewed bool `json:"privacy_controls_reviewed"` + PilotRecipientsConfirmed bool `json:"pilot_recipients_confirmed"` +} + +// Authorization binds one short-lived live-send decision to a specific +// aggregate preflight and sender identity. It intentionally contains no +// recipient addresses or unsubscribe URLs. +type Authorization struct { + SchemaVersion string `json:"schema_version"` + Tool string `json:"tool"` + Status string `json:"status"` + Channel string `json:"channel"` + Provider string `json:"provider"` + EditionID string `json:"edition_id"` + Audience string `json:"audience"` + Sender string `json:"sender"` + ReplyTo string `json:"reply_to,omitempty"` + AuthorizedBy string `json:"authorized_by"` + AuthorizedAt string `json:"authorized_at"` + ExpiresAt string `json:"expires_at"` + PreflightSHA256 string `json:"preflight_sha256"` + ManifestSHA256 string `json:"manifest_sha256"` + ApprovalSHA256 string `json:"approval_sha256"` + SubscriberSHA256 string `json:"subscriber_csv_sha256"` + SuppressionSHA256 string `json:"suppression_csv_sha256"` + EmailSHA256 string `json:"email_template_sha256"` + EligibleRecipients int `json:"eligible_recipients"` + MaxRecipients int `json:"max_recipients"` + Attestations Attestations `json:"attestations"` + DeliveryAuthorized bool `json:"delivery_authorized"` + FileSHA256 string `json:"-"` +} + +type AuthorizationRequest struct { + PreflightPath string + Provider string + Sender string + ReplyTo string + AuthorizedBy string + AuthorizedAt time.Time + ExpiresAt time.Time + Attestations Attestations +} + +func Authorize(request AuthorizationRequest) (Authorization, []byte, error) { + preflightRaw, err := readPrivateRegularFile(request.PreflightPath, "preflight") + if err != nil { + return Authorization{}, nil, err + } + plan, err := decodePreflight(preflightRaw) + if err != nil { + return Authorization{}, nil, err + } + if err := validatePreflightForAuthorization(plan); err != nil { + return Authorization{}, nil, err + } + provider := strings.ToLower(strings.TrimSpace(request.Provider)) + if provider != "resend" { + return Authorization{}, nil, errors.New("provider must be resend for the current pilot adapter") + } + sender, err := canonicalMailbox(request.Sender, "sender") + if err != nil { + return Authorization{}, nil, err + } + replyTo := "" + if strings.TrimSpace(request.ReplyTo) != "" { + replyTo, err = canonicalMailbox(request.ReplyTo, "reply-to") + if err != nil { + return Authorization{}, nil, err + } + } + operator := strings.TrimSpace(request.AuthorizedBy) + if operator == "" || len([]rune(operator)) > 120 || strings.ContainsAny(operator, "\r\n") { + return Authorization{}, nil, errors.New("authorized-by is required and must be a single bounded label") + } + if request.AuthorizedAt.IsZero() || request.ExpiresAt.IsZero() { + return Authorization{}, nil, errors.New("authorization and expiry times are required") + } + authorizedAt := request.AuthorizedAt.UTC() + expiresAt := request.ExpiresAt.UTC() + if !expiresAt.After(authorizedAt) || expiresAt.Sub(authorizedAt) > maximumAuthorizationAge { + return Authorization{}, nil, errors.New("launch authorization must expire after authorization and within one hour") + } + preflightAt, err := time.Parse(time.RFC3339, plan.GeneratedAt) + if err != nil || authorizedAt.Before(preflightAt) { + return Authorization{}, nil, errors.New("launch authorization cannot predate the delivery preflight") + } + if !allAttested(request.Attestations) { + return Authorization{}, nil, errors.New("all launch attestations are required") + } + digest := sha256.Sum256(preflightRaw) + authorization := Authorization{ + SchemaVersion: authorizationSchemaVersion, + Tool: authorizationToolVersion, + Status: "live_send_authorized", + Channel: "email", + Provider: provider, + EditionID: plan.EditionID, + Audience: plan.Audience, + Sender: sender, + ReplyTo: replyTo, + AuthorizedBy: operator, + AuthorizedAt: authorizedAt.Format(time.RFC3339), + ExpiresAt: expiresAt.Format(time.RFC3339), + PreflightSHA256: hex.EncodeToString(digest[:]), + ManifestSHA256: plan.ManifestSHA256, + ApprovalSHA256: plan.ApprovalSHA256, + SubscriberSHA256: plan.Sources.SubscriberCSVSHA256, + SuppressionSHA256: plan.Sources.SuppressionCSVSHA256, + EmailSHA256: plan.EmailTemplateSHA256, + EligibleRecipients: plan.Counts.Eligible, + MaxRecipients: plan.MaxRecipients, + Attestations: request.Attestations, + DeliveryAuthorized: true, + } + raw, err := json.MarshalIndent(authorization, "", " ") + if err != nil { + return Authorization{}, nil, fmt.Errorf("encode launch authorization: %w", err) + } + return authorization, append(raw, '\n'), nil +} + +func WriteAuthorization(path string, raw []byte) error { + return writePrivateExclusive(path, raw, "launch authorization") +} + +func LoadAuthorization(path string, preflightRaw []byte, sendAt time.Time) (Authorization, error) { + raw, err := readPrivateRegularFile(path, "launch authorization") + if err != nil { + return Authorization{}, err + } + var authorization Authorization + if err := decodeStrictJSON(raw, &authorization); err != nil { + return Authorization{}, fmt.Errorf("decode launch authorization: %w", err) + } + if authorization.SchemaVersion != authorizationSchemaVersion || authorization.Tool != authorizationToolVersion || authorization.Status != "live_send_authorized" || authorization.Channel != "email" || authorization.Provider != "resend" || !authorization.DeliveryAuthorized { + return Authorization{}, errors.New("launch authorization has an unsupported contract or inactive status") + } + if !allAttested(authorization.Attestations) { + return Authorization{}, errors.New("launch authorization is missing required attestations") + } + if _, err := canonicalMailbox(authorization.Sender, "sender"); err != nil { + return Authorization{}, err + } + if authorization.ReplyTo != "" { + if _, err := canonicalMailbox(authorization.ReplyTo, "reply-to"); err != nil { + return Authorization{}, err + } + } + authorizedAt, err := time.Parse(time.RFC3339, authorization.AuthorizedAt) + if err != nil { + return Authorization{}, errors.New("launch authorization time is invalid") + } + expiresAt, err := time.Parse(time.RFC3339, authorization.ExpiresAt) + if err != nil || !expiresAt.After(authorizedAt) || expiresAt.Sub(authorizedAt) > maximumAuthorizationAge { + return Authorization{}, errors.New("launch authorization expiry is invalid") + } + if sendAt.IsZero() || sendAt.UTC().Before(authorizedAt) || sendAt.UTC().After(expiresAt) { + return Authorization{}, errors.New("send time is outside the launch authorization window") + } + digest := sha256.Sum256(preflightRaw) + if authorization.PreflightSHA256 != hex.EncodeToString(digest[:]) { + return Authorization{}, errors.New("launch authorization does not match the supplied preflight") + } + authorizationDigest := sha256.Sum256(raw) + authorization.FileSHA256 = hex.EncodeToString(authorizationDigest[:]) + return authorization, nil +} + +func VerifyLivePlan(authorization Authorization, original, live deliverypreflight.Plan) error { + if original.EditionID != authorization.EditionID || live.EditionID != authorization.EditionID || original.Audience != authorization.Audience || live.Audience != authorization.Audience { + return errors.New("live preflight edition or audience differs from launch authorization") + } + if original.ManifestSHA256 != authorization.ManifestSHA256 || live.ManifestSHA256 != authorization.ManifestSHA256 || original.ApprovalSHA256 != authorization.ApprovalSHA256 || live.ApprovalSHA256 != authorization.ApprovalSHA256 || original.EmailTemplateSHA256 != authorization.EmailSHA256 || live.EmailTemplateSHA256 != authorization.EmailSHA256 { + return errors.New("live content digests differ from launch authorization") + } + if original.Sources.SubscriberCSVSHA256 != authorization.SubscriberSHA256 || live.Sources.SubscriberCSVSHA256 != authorization.SubscriberSHA256 || original.Sources.SuppressionCSVSHA256 != authorization.SuppressionSHA256 || live.Sources.SuppressionCSVSHA256 != authorization.SuppressionSHA256 { + return errors.New("live consent or suppression inputs differ from launch authorization") + } + if original.Counts.Eligible != authorization.EligibleRecipients || live.Counts.Eligible != authorization.EligibleRecipients || original.MaxRecipients != authorization.MaxRecipients || live.MaxRecipients != authorization.MaxRecipients { + return errors.New("live recipient counts or pilot ceiling differ from launch authorization") + } + if !live.ConsentValidated || !live.SuppressionApplied || !live.UnsubscribeURLsValidated || live.ContainsRecipientAddresses || live.DeliveryAuthorized { + return errors.New("live preflight safety invariants are not satisfied") + } + return nil +} + +func ReadPreflight(path string) (deliverypreflight.Plan, []byte, error) { + raw, err := readPrivateRegularFile(path, "preflight") + if err != nil { + return deliverypreflight.Plan{}, nil, err + } + plan, err := decodePreflight(raw) + if err != nil { + return deliverypreflight.Plan{}, nil, err + } + return plan, raw, nil +} + +func decodePreflight(raw []byte) (deliverypreflight.Plan, error) { + var plan deliverypreflight.Plan + if err := decodeStrictJSON(raw, &plan); err != nil { + return deliverypreflight.Plan{}, fmt.Errorf("decode delivery preflight: %w", err) + } + return plan, nil +} + +func validatePreflightForAuthorization(plan deliverypreflight.Plan) error { + if plan.SchemaVersion != "1.0" || plan.Status != "consent_preflight_passed" || plan.Channel != "email" { + return errors.New("delivery preflight has an unsupported contract or status") + } + if !plan.ConsentValidated || !plan.SuppressionApplied || !plan.UnsubscribeURLsValidated || plan.ContainsRecipientAddresses || plan.ProviderConfigured || plan.DeliveryAuthorized { + return errors.New("delivery preflight safety invariants are not satisfied") + } + if plan.Counts.Eligible < 1 || plan.Counts.Eligible > plan.MaxRecipients || plan.MaxRecipients < 1 { + return errors.New("delivery preflight recipient counts are invalid") + } + if plan.EditionID == "" || plan.Audience == "" || plan.ManifestSHA256 == "" || plan.ApprovalSHA256 == "" || plan.Sources.SubscriberCSVSHA256 == "" || plan.Sources.SuppressionCSVSHA256 == "" || plan.EmailTemplateSHA256 == "" { + return errors.New("delivery preflight is missing required identities or digests") + } + if _, err := time.Parse(time.RFC3339, plan.GeneratedAt); err != nil { + return errors.New("delivery preflight generation time is invalid") + } + return nil +} + +func canonicalMailbox(value, label string) (string, error) { + value = strings.TrimSpace(value) + if value == "" || len(value) > 320 || strings.ContainsAny(value, "\r\n") { + return "", fmt.Errorf("%s mailbox is empty or malformed", label) + } + parsed, err := mail.ParseAddress(value) + if err != nil || parsed.Address == "" { + return "", fmt.Errorf("%s mailbox is malformed", label) + } + return parsed.String(), nil +} + +func allAttested(value Attestations) bool { + return value.SenderDomainAuthenticated && value.BounceComplaintReady && value.PrivacyControlsReviewed && value.PilotRecipientsConfirmed +} + +func decodeStrictJSON(raw []byte, target any) error { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return err + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return errors.New("JSON contains trailing content") + } + return nil +} + +func readPrivateRegularFile(path, label string) ([]byte, error) { + path = strings.TrimSpace(path) + if path == "" { + return nil, fmt.Errorf("%s path is required", label) + } + info, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("inspect %s: %w", label, err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("%s must be a regular non-symlink file", label) + } + if info.Size() > maximumPrivateFileSize { + return nil, fmt.Errorf("%s exceeds the private file size limit", label) + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", label, err) + } + return raw, nil +} + +func writePrivateExclusive(path string, raw []byte, label string) error { + path = strings.TrimSpace(path) + if path == "" { + return fmt.Errorf("%s output path is required", label) + } + absolute, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("resolve %s output: %w", label, err) + } + parent := filepath.Dir(absolute) + if err := os.MkdirAll(parent, 0o700); err != nil { + return fmt.Errorf("create %s output directory: %w", label, err) + } + resolvedParent, err := filepath.EvalSymlinks(parent) + if err != nil { + return fmt.Errorf("resolve %s output directory: %w", label, err) + } + target := filepath.Join(resolvedParent, filepath.Base(absolute)) + file, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("%s output already exists", label) + } + return fmt.Errorf("create %s output: %w", label, err) + } + writeErr := func() error { + if _, err := file.Write(raw); err != nil { + return err + } + return file.Sync() + }() + closeErr := file.Close() + if writeErr != nil || closeErr != nil { + _ = os.Remove(target) + if writeErr != nil { + return fmt.Errorf("write %s: %w", label, writeErr) + } + return fmt.Errorf("close %s: %w", label, closeErr) + } + return nil +} diff --git a/internal/emaildelivery/delivery.go b/internal/emaildelivery/delivery.go new file mode 100644 index 0000000..2b52a87 --- /dev/null +++ b/internal/emaildelivery/delivery.go @@ -0,0 +1,222 @@ +package emaildelivery + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "tradegravity/internal/deliverypreflight" +) + +const unsubscribePlaceholder = "{{UNSUBSCRIBE_URL}}" + +type Provider interface { + Name() string + Send(context.Context, Message, string) (string, error) +} + +type Message struct { + From string + ReplyTo string + To string + Subject string + HTML string + Text string + ListUnsubscribe string + ListUnsubscribePost string +} + +type DeliveryRequest struct { + KitDir string + SubscriberCSV string + SuppressionCSV string + PreflightPath string + AuthorizationPath string + LedgerPath string + LedgerSecret []byte + SendAt time.Time + Provider Provider + SendLive bool +} + +type DeliveryResult struct { + EditionID string + Audience string + Eligible int + Accepted int + Skipped int + Pending int +} + +func Deliver(ctx context.Context, request DeliveryRequest) (DeliveryResult, error) { + if !request.SendLive { + return DeliveryResult{}, errors.New("live delivery requires the explicit send-live acknowledgement") + } + if request.Provider == nil || request.Provider.Name() != "resend" { + return DeliveryResult{}, errors.New("the authorized resend provider is required") + } + originalPlan, preflightRaw, err := ReadPreflight(request.PreflightPath) + if err != nil { + return DeliveryResult{}, err + } + authorization, err := LoadAuthorization(request.AuthorizationPath, preflightRaw, request.SendAt) + if err != nil { + return DeliveryResult{}, err + } + if authorization.Provider != request.Provider.Name() { + return DeliveryResult{}, errors.New("configured provider does not match launch authorization") + } + live, err := deliverypreflight.Build(deliverypreflight.Request{ + KitDir: request.KitDir, + SubscriberCSV: request.SubscriberCSV, + SuppressionCSV: request.SuppressionCSV, + GeneratedAt: request.SendAt.UTC(), + MaxRecipients: authorization.MaxRecipients, + }) + if err != nil { + return DeliveryResult{}, fmt.Errorf("rerun live delivery preflight: %w", err) + } + if err := VerifyLivePlan(authorization, originalPlan, live.Plan); err != nil { + return DeliveryResult{}, err + } + template, err := loadEmailTemplate(request.KitDir) + if err != nil { + return DeliveryResult{}, err + } + ledger, err := OpenLedger(request.LedgerPath, request.LedgerSecret) + if err != nil { + return DeliveryResult{}, err + } + defer ledger.Close() + + result := DeliveryResult{ + EditionID: authorization.EditionID, + Audience: authorization.Audience, + Eligible: len(live.EligibleRecipients), + } + for _, recipient := range live.EligibleRecipients { + message, digest, err := renderMessage(template, authorization, recipient) + if err != nil { + return result, err + } + prepared, err := ledger.Prepare(ctx, DeliveryAttempt{ + EditionID: authorization.EditionID, + Audience: authorization.Audience, + Email: recipient.Email, + Provider: request.Provider.Name(), + ContentSHA256: digest, + AuthorizationSHA256: authorization.FileSHA256, + AttemptedAt: request.SendAt.UTC(), + }) + if err != nil { + pending, accepted, countErr := ledger.Counts(ctx, authorization.EditionID, authorization.Audience) + if countErr == nil { + result.Pending = pending + result.Accepted = accepted + } + return result, fmt.Errorf("prepare recipient delivery: %w", err) + } + if prepared.AlreadyAccepted { + result.Skipped++ + continue + } + messageID, err := request.Provider.Send(ctx, message, prepared.IdempotencyKey) + if err != nil { + result.Pending++ + return result, fmt.Errorf("provider did not confirm delivery acceptance; pending ledger entry requires reconciliation: %w", err) + } + if err := ledger.MarkAccepted(ctx, prepared.DeliveryKey, messageID, request.SendAt.UTC()); err != nil { + result.Pending++ + return result, fmt.Errorf("provider accepted delivery but local ledger update failed; reconcile before retry: %w", err) + } + result.Accepted++ + } + pending, accepted, err := ledger.Counts(ctx, authorization.EditionID, authorization.Audience) + if err != nil { + return result, fmt.Errorf("count delivery ledger: %w", err) + } + result.Pending = pending + result.Accepted = accepted + return result, nil +} + +type emailTemplate struct { + Subject string + HTML string + Text string +} + +func loadEmailTemplate(kitDir string) (emailTemplate, error) { + root, err := filepath.Abs(strings.TrimSpace(kitDir)) + if err != nil || strings.TrimSpace(kitDir) == "" { + return emailTemplate{}, errors.New("distribution-kit directory is required") + } + read := func(relative string) ([]byte, error) { + path := filepath.Join(root, filepath.FromSlash(relative)) + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Size() > maximumPrivateFileSize { + return nil, fmt.Errorf("approved email file %s is unavailable or unsafe", relative) + } + return os.ReadFile(path) + } + subjectRaw, err := read("email/subject.txt") + if err != nil { + return emailTemplate{}, err + } + htmlRaw, err := read("email/body.html") + if err != nil { + return emailTemplate{}, err + } + textRaw, err := read("email/body.md") + if err != nil { + return emailTemplate{}, err + } + subject := strings.TrimSpace(string(subjectRaw)) + if subject == "" || len([]rune(subject)) > 200 || strings.ContainsAny(subject, "\r\n") { + return emailTemplate{}, errors.New("approved email subject is empty, too long, or multiline") + } + if bytes.Count(htmlRaw, []byte(unsubscribePlaceholder)) != 1 || bytes.Count(textRaw, []byte(unsubscribePlaceholder)) != 1 { + return emailTemplate{}, errors.New("approved email bodies must each contain exactly one unsubscribe placeholder") + } + return emailTemplate{Subject: subject, HTML: string(htmlRaw), Text: string(textRaw)}, nil +} + +func renderMessage(template emailTemplate, authorization Authorization, recipient deliverypreflight.Recipient) (Message, string, error) { + if recipient.Email == "" || recipient.UnsubscribeURL == "" || strings.ContainsAny(recipient.UnsubscribeURL, "\r\n") { + return Message{}, "", errors.New("eligible recipient is missing a safe address or unsubscribe URL") + } + htmlBody := strings.Replace(template.HTML, unsubscribePlaceholder, recipient.UnsubscribeURL, 1) + textBody := strings.Replace(template.Text, unsubscribePlaceholder, recipient.UnsubscribeURL, 1) + if strings.Contains(htmlBody, unsubscribePlaceholder) || strings.Contains(textBody, unsubscribePlaceholder) { + return Message{}, "", errors.New("unsubscribe placeholder remained after recipient rendering") + } + message := Message{ + From: authorization.Sender, + ReplyTo: authorization.ReplyTo, + To: recipient.Email, + Subject: template.Subject, + HTML: htmlBody, + Text: textBody, + ListUnsubscribe: "<" + recipient.UnsubscribeURL + ">", + ListUnsubscribePost: "List-Unsubscribe=One-Click", + } + digestSource := strings.Join([]string{ + message.From, + message.ReplyTo, + message.To, + message.Subject, + message.HTML, + message.Text, + message.ListUnsubscribe, + message.ListUnsubscribePost, + }, "\x00") + digest := sha256.Sum256([]byte(digestSource)) + return message, hex.EncodeToString(digest[:]), nil +} diff --git a/internal/emaildelivery/emaildelivery_test.go b/internal/emaildelivery/emaildelivery_test.go new file mode 100644 index 0000000..d357655 --- /dev/null +++ b/internal/emaildelivery/emaildelivery_test.go @@ -0,0 +1,546 @@ +package emaildelivery + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "tradegravity/internal/deliverypreflight" + "tradegravity/internal/distributionkit" +) + +func TestAuthorizationBindsPreflightWithoutRecipientData(t *testing.T) { + fixture := newDeliveryFixture(t) + authorization, raw, err := Authorize(fixture.authorizationRequest()) + if err != nil { + t.Fatal(err) + } + if !authorization.DeliveryAuthorized || authorization.EligibleRecipients != 2 || authorization.Provider != "resend" { + t.Fatalf("unexpected authorization: %+v", authorization) + } + if bytes.Contains(raw, []byte("example.invalid")) || bytes.Contains(raw, []byte("token=")) { + t.Fatal("launch authorization leaked a recipient address or token") + } + path := filepath.Join(fixture.root, "launch.json") + if err := WriteAuthorization(path, raw); err != nil { + t.Fatal(err) + } + if err := WriteAuthorization(path, raw); err == nil { + t.Fatal("WriteAuthorization() overwrote an existing authorization") + } + preflightRaw, err := os.ReadFile(fixture.preflightPath) + if err != nil { + t.Fatal(err) + } + loaded, err := LoadAuthorization(path, preflightRaw, fixture.sendAt) + if err != nil || loaded.PreflightSHA256 != authorization.PreflightSHA256 { + t.Fatalf("LoadAuthorization() = %+v, %v", loaded, err) + } + + tampered := append([]byte(nil), preflightRaw...) + tampered = append(tampered, ' ') + if _, err := LoadAuthorization(path, tampered, fixture.sendAt); err == nil { + t.Fatal("LoadAuthorization() accepted a changed preflight") + } +} + +func TestAuthorizationRejectsMissingControlsAndLongWindows(t *testing.T) { + fixture := newDeliveryFixture(t) + request := fixture.authorizationRequest() + request.Attestations.BounceComplaintReady = false + if _, _, err := Authorize(request); err == nil { + t.Fatal("Authorize() accepted a missing feedback-control attestation") + } + request = fixture.authorizationRequest() + request.ExpiresAt = request.AuthorizedAt.Add(time.Hour + time.Second) + if _, _, err := Authorize(request); err == nil { + t.Fatal("Authorize() accepted a window longer than one hour") + } + request = fixture.authorizationRequest() + request.Sender = "bad\r\nBcc: victim@example.invalid" + if _, _, err := Authorize(request); err == nil { + t.Fatal("Authorize() accepted sender header injection") + } +} + +func TestDeliverSendsOnceWithOneClickHeadersAndPrivateLedger(t *testing.T) { + fixture := newDeliveryFixture(t) + authorizationPath := fixture.writeAuthorization(t) + provider := &recordingProvider{} + request := fixture.deliveryRequest(authorizationPath, provider) + + first, err := Deliver(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if first.Eligible != 2 || first.Accepted != 2 || first.Skipped != 0 || first.Pending != 0 || len(provider.messages) != 2 { + t.Fatalf("unexpected first delivery: result=%+v provider_calls=%d", first, len(provider.messages)) + } + for _, message := range provider.messages { + if strings.Contains(message.HTML, unsubscribePlaceholder) || strings.Contains(message.Text, unsubscribePlaceholder) { + t.Fatal("provider message retained the unsubscribe placeholder") + } + if !strings.HasPrefix(message.ListUnsubscribe, "") || message.ListUnsubscribePost != "List-Unsubscribe=One-Click" { + t.Fatalf("invalid one-click headers: %+v", message) + } + if strings.Count(message.HTML, "token=") != 1 || strings.Count(message.Text, "token=") != 1 { + t.Fatal("visible recipient unsubscribe link was not rendered exactly once per body") + } + } + if provider.idempotencyKeys[0] == provider.idempotencyKeys[1] { + t.Fatal("different recipients shared an idempotency key") + } + + second, err := Deliver(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if second.Accepted != 2 || second.Skipped != 2 || second.Pending != 0 || len(provider.messages) != 2 { + t.Fatalf("accepted deliveries were sent again: result=%+v provider_calls=%d", second, len(provider.messages)) + } + for _, suffix := range []string{"", "-wal", "-shm"} { + raw, err := os.ReadFile(fixture.ledgerPath + suffix) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + t.Fatal(err) + } + if bytes.Contains(raw, []byte("example.invalid")) || bytes.Contains(raw, []byte("token=")) { + t.Fatalf("delivery ledger %s leaked a recipient address or token", suffix) + } + } +} + +func TestDeliverLeavesUnknownProviderResultPendingAndRefusesRetry(t *testing.T) { + fixture := newDeliveryFixture(t) + authorizationPath := fixture.writeAuthorization(t) + provider := &recordingProvider{fail: true} + request := fixture.deliveryRequest(authorizationPath, provider) + + result, err := Deliver(context.Background(), request) + if err == nil || result.Pending != 1 || len(provider.messages) != 1 { + t.Fatalf("unknown provider result was not left pending: result=%+v err=%v calls=%d", result, err, len(provider.messages)) + } + provider.fail = false + result, err = Deliver(context.Background(), request) + if !errors.Is(err, ErrDeliveryPending) || len(provider.messages) != 1 { + t.Fatalf("pending delivery was retried: result=%+v err=%v calls=%d", result, err, len(provider.messages)) + } +} + +func TestReconcileNotAcceptedRequiresNewAuthorizationBeforeRetry(t *testing.T) { + fixture := newDeliveryFixture(t) + authorizationPath := fixture.writeAuthorization(t) + provider := &recordingProvider{fail: true} + request := fixture.deliveryRequest(authorizationPath, provider) + if _, err := Deliver(context.Background(), request); err == nil { + t.Fatal("initial synthetic provider failure was accepted") + } + + ledger := fixture.openLedger(t) + resolvedAt := fixture.sendAt.Add(time.Minute) + reconciled, err := ledger.Reconcile(context.Background(), ReconciliationRequest{ + EditionID: fixture.editionID(t), Audience: "pilot-audience", Email: "alpha@example.invalid", + Outcome: "not_accepted", ResolvedBy: "maintainer", Evidence: "resend-dashboard-no-message", + ResolvedAt: resolvedAt, + }) + if err != nil || !reconciled.Changed { + t.Fatalf("Reconcile(not_accepted) = %+v, %v", reconciled, err) + } + if err := ledger.Close(); err != nil { + t.Fatal(err) + } + + provider.fail = false + request.SendAt = resolvedAt.Add(30 * time.Second) + if _, err := Deliver(context.Background(), request); !errors.Is(err, ErrNewAuthorizationRequired) { + t.Fatalf("same authorization retried reconciled delivery: %v", err) + } + if len(provider.messages) != 1 { + t.Fatalf("provider called again under same authorization: %d", len(provider.messages)) + } + + retryAuthorizedAt := fixture.sendAt.Add(2 * time.Minute) + retryAuthorization := fixture.writeAuthorizationAt(t, "launch-retry.json", retryAuthorizedAt, retryAuthorizedAt.Add(20*time.Minute)) + request.AuthorizationPath = retryAuthorization + request.SendAt = retryAuthorizedAt.Add(time.Minute) + result, err := Deliver(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if result.Accepted != 2 || result.Pending != 0 || len(provider.messages) != 3 { + t.Fatalf("new authorization did not complete retry and remaining recipient: result=%+v calls=%d", result, len(provider.messages)) + } + if provider.idempotencyKeys[0] != provider.idempotencyKeys[1] { + t.Fatal("reconciled retry changed the stable provider idempotency key") + } +} + +func TestReconcileAcceptedSkipsProviderRetry(t *testing.T) { + fixture := newDeliveryFixture(t) + authorizationPath := fixture.writeAuthorization(t) + provider := &recordingProvider{fail: true} + request := fixture.deliveryRequest(authorizationPath, provider) + if _, err := Deliver(context.Background(), request); err == nil { + t.Fatal("initial synthetic provider failure was accepted") + } + + ledger := fixture.openLedger(t) + reconciled, err := ledger.Reconcile(context.Background(), ReconciliationRequest{ + EditionID: fixture.editionID(t), Audience: "pilot-audience", Email: "alpha@example.invalid", + Outcome: "accepted", ProviderMessageID: "provider-confirmed-1", ResolvedBy: "maintainer", + Evidence: "resend-dashboard-message-match", ResolvedAt: fixture.sendAt.Add(time.Minute), + }) + if err != nil || !reconciled.Changed { + t.Fatalf("Reconcile(accepted) = %+v, %v", reconciled, err) + } + repeated, err := ledger.Reconcile(context.Background(), ReconciliationRequest{ + EditionID: fixture.editionID(t), Audience: "pilot-audience", Email: "alpha@example.invalid", + Outcome: "accepted", ProviderMessageID: "provider-confirmed-1", ResolvedBy: "maintainer", + Evidence: "resend-dashboard-message-match", ResolvedAt: fixture.sendAt.Add(time.Minute), + }) + if err != nil || !repeated.AlreadyResolved { + t.Fatalf("repeated accepted reconciliation = %+v, %v", repeated, err) + } + if err := ledger.Close(); err != nil { + t.Fatal(err) + } + + provider.fail = false + request.SendAt = fixture.sendAt.Add(2 * time.Minute) + result, err := Deliver(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if result.Accepted != 2 || result.Skipped != 1 || len(provider.messages) != 2 { + t.Fatalf("accepted reconciliation was resent: result=%+v calls=%d", result, len(provider.messages)) + } +} + +func TestReconcileRejectsPIIInAuditFieldsAndContradictoryOutcome(t *testing.T) { + fixture := newDeliveryFixture(t) + authorizationPath := fixture.writeAuthorization(t) + provider := &recordingProvider{fail: true} + request := fixture.deliveryRequest(authorizationPath, provider) + if _, err := Deliver(context.Background(), request); err == nil { + t.Fatal("initial synthetic provider failure was accepted") + } + ledger := fixture.openLedger(t) + defer ledger.Close() + base := ReconciliationRequest{ + EditionID: fixture.editionID(t), Audience: "pilot-audience", Email: "alpha@example.invalid", + Outcome: "not_accepted", ResolvedBy: "maintainer", Evidence: "dashboard-check", + ResolvedAt: fixture.sendAt.Add(time.Minute), + } + withPII := base + withPII.Evidence = "reader@example.invalid" + if _, err := ledger.Reconcile(context.Background(), withPII); err == nil { + t.Fatal("Reconcile() accepted PII in the stored evidence label") + } + contradictory := base + contradictory.ProviderMessageID = "provider-id" + if _, err := ledger.Reconcile(context.Background(), contradictory); err == nil { + t.Fatal("Reconcile() accepted a message ID for a not_accepted outcome") + } + acceptedWithoutID := base + acceptedWithoutID.Outcome = "accepted" + if _, err := ledger.Reconcile(context.Background(), acceptedWithoutID); err == nil { + t.Fatal("Reconcile() accepted an accepted outcome without a provider message ID") + } +} + +func TestOpenLedgerAddsReconciliationColumnsToEarlierPilotSchema(t *testing.T) { + path := filepath.Join(t.TempDir(), "delivery-ledger.db") + database, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + _, err = database.Exec(`CREATE TABLE deliveries ( + delivery_key TEXT PRIMARY KEY, + edition_id TEXT NOT NULL, + audience TEXT NOT NULL, + recipient_key TEXT NOT NULL, + provider TEXT NOT NULL, + content_sha256 TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending', 'accepted')), + attempted_at TEXT NOT NULL, + accepted_at TEXT, + provider_message_id TEXT, + UNIQUE(edition_id, audience, recipient_key) + )`) + if err != nil { + t.Fatal(err) + } + if err := database.Close(); err != nil { + t.Fatal(err) + } + ledger, err := OpenLedger(path, bytes.Repeat([]byte("m"), 32)) + if err != nil { + t.Fatal(err) + } + defer ledger.Close() + rows, err := ledger.db.Query(`PRAGMA table_info(deliveries)`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + found := map[string]bool{} + for rows.Next() { + var cid int + var name, columnType string + var notNull, primaryKey int + var defaultValue any + if err := rows.Scan(&cid, &name, &columnType, ¬Null, &defaultValue, &primaryKey); err != nil { + t.Fatal(err) + } + found[name] = true + } + for _, name := range []string{"authorization_sha256", "resolution", "resolved_at", "resolved_by", "resolution_evidence"} { + if !found[name] { + t.Fatalf("migrated ledger omitted %s", name) + } + } +} + +func TestDeliverRequiresExplicitLiveAcknowledgement(t *testing.T) { + _, err := Deliver(context.Background(), DeliveryRequest{}) + if err == nil || !strings.Contains(err.Error(), "send-live") { + t.Fatalf("Deliver() without live acknowledgement = %v", err) + } +} + +func TestResendProviderRequestContractAndBoundedErrors(t *testing.T) { + var received struct { + Authorization string + Idempotency string + Payload map[string]any + } + server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + received.Authorization = request.Header.Get("Authorization") + received.Idempotency = request.Header.Get("Idempotency-Key") + raw, _ := io.ReadAll(request.Body) + _ = json.Unmarshal(raw, &received.Payload) + response.Header().Set("Content-Type", "application/json") + _, _ = response.Write([]byte(`{"id":"provider-message-1"}`)) + })) + defer server.Close() + provider, err := NewResendProvider("secret-test-key", server.Client()) + if err != nil { + t.Fatal(err) + } + provider.endpoint = server.URL + message := Message{ + From: "TradeGravity ", To: "reader@example.invalid", Subject: "Pilot", + HTML: "

Body

", Text: "Body", ListUnsubscribe: "", ListUnsubscribePost: "List-Unsubscribe=One-Click", + } + id, err := provider.Send(context.Background(), message, "tradegravity/edition/key") + if err != nil || id != "provider-message-1" { + t.Fatalf("provider.Send() = %q, %v", id, err) + } + if received.Authorization != "Bearer secret-test-key" || received.Idempotency != "tradegravity/edition/key" { + t.Fatalf("provider authentication or idempotency header missing: %+v", received) + } + headers, ok := received.Payload["headers"].(map[string]any) + if !ok || headers["List-Unsubscribe"] != message.ListUnsubscribe || headers["List-Unsubscribe-Post"] != message.ListUnsubscribePost { + t.Fatalf("provider payload omitted one-click headers: %+v", received.Payload) + } + to, ok := received.Payload["to"].([]any) + if !ok || len(to) != 1 || to[0] != message.To { + t.Fatalf("provider payload did not isolate the recipient: %+v", received.Payload) + } + received.Payload = nil + confirmation := message + confirmation.ListUnsubscribe = "" + confirmation.ListUnsubscribePost = "" + if _, err := provider.Send(context.Background(), confirmation, "tradegravity-confirm/test"); err != nil { + t.Fatal(err) + } + if _, exists := received.Payload["headers"]; exists { + t.Fatalf("pre-consent confirmation payload contained unsubscribe headers: %+v", received.Payload) + } + confirmation.ListUnsubscribe = "" + if _, err := provider.Send(context.Background(), confirmation, "tradegravity-confirm/test-2"); err == nil { + t.Fatal("provider accepted only one of the paired unsubscribe headers") + } + + errorServer := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte(`{"message":"private recipient detail"}`)) + })) + defer errorServer.Close() + provider.client = errorServer.Client() + provider.endpoint = errorServer.URL + _, err = provider.Send(context.Background(), message, "tradegravity/edition/key") + if err == nil || strings.Contains(err.Error(), "private recipient detail") { + t.Fatalf("provider error leaked its response body: %v", err) + } +} + +type recordingProvider struct { + messages []Message + idempotencyKeys []string + fail bool +} + +func (provider *recordingProvider) Name() string { return "resend" } + +func (provider *recordingProvider) Send(_ context.Context, message Message, idempotencyKey string) (string, error) { + provider.messages = append(provider.messages, message) + provider.idempotencyKeys = append(provider.idempotencyKeys, idempotencyKey) + if provider.fail { + return "", errors.New("synthetic unknown result") + } + return "provider-id-" + string(rune('a'+len(provider.messages)-1)), nil +} + +type deliveryFixture struct { + root string + kitDir string + subscriberPath string + suppressionPath string + preflightPath string + ledgerPath string + preflightAt time.Time + authorizedAt time.Time + sendAt time.Time +} + +func newDeliveryFixture(t *testing.T) deliveryFixture { + t.Helper() + root := t.TempDir() + briefing, err := os.ReadFile(filepath.Join("..", "..", "examples", "sample-data", "briefing.json")) + if err != nil { + t.Fatal(err) + } + bundle, err := distributionkit.Build(briefing, "https://example.org/TradeGravity/") + if err != nil { + t.Fatal(err) + } + kitDir := filepath.Join(root, "kit") + if err := distributionkit.Write(kitDir, bundle); err != nil { + t.Fatal(err) + } + _, approvalRaw, err := distributionkit.Approve(kitDir, distributionkit.ApprovalRequest{ + Reviewer: "reviewer", Audience: "pilot-audience", Channels: []string{"email"}, + ApprovedAt: time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC), Attested: true, + }) + if err != nil { + t.Fatal(err) + } + if err := distributionkit.WriteApproval(kitDir, approvalRaw); err != nil { + t.Fatal(err) + } + subscriberPath := filepath.Join(root, "private", "subscribers.csv") + suppressionPath := filepath.Join(root, "private", "suppressions.csv") + if err := os.MkdirAll(filepath.Dir(subscriberPath), 0o700); err != nil { + t.Fatal(err) + } + subscribers := strings.Join([]string{ + "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url", + "alpha@example.invalid,pilot-audience,active,2026-07-10T01:00:00Z,double_opt_in,test-form,v1,https://subscriptions.example.invalid/u?token=opaque-alpha", + "beta@example.invalid,pilot-audience,active,2026-07-11T01:00:00Z,double_opt_in,test-form,v1,https://subscriptions.example.invalid/u?token=opaque-beta", + "", + }, "\n") + if err := os.WriteFile(subscriberPath, []byte(subscribers), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(suppressionPath, []byte("email,reason,suppressed_at\n"), 0o600); err != nil { + t.Fatal(err) + } + preflightAt := time.Date(2026, 7, 17, 12, 30, 0, 0, time.UTC) + preflight, err := deliverypreflight.Build(deliverypreflight.Request{ + KitDir: kitDir, SubscriberCSV: subscriberPath, SuppressionCSV: suppressionPath, + GeneratedAt: preflightAt, MaxRecipients: 25, + }) + if err != nil { + t.Fatal(err) + } + preflightPath := filepath.Join(root, "delivery-preflight.json") + if err := deliverypreflight.Write(preflightPath, kitDir, preflight.JSON); err != nil { + t.Fatal(err) + } + return deliveryFixture{ + root: root, kitDir: kitDir, subscriberPath: subscriberPath, suppressionPath: suppressionPath, + preflightPath: preflightPath, ledgerPath: filepath.Join(root, "private", "delivery-ledger.db"), + preflightAt: preflightAt, authorizedAt: preflightAt.Add(time.Minute), sendAt: preflightAt.Add(2 * time.Minute), + } +} + +func (fixture deliveryFixture) authorizationRequest() AuthorizationRequest { + return AuthorizationRequest{ + PreflightPath: fixture.preflightPath, Provider: "resend", Sender: "TradeGravity ", + ReplyTo: "maintainer@example.org", AuthorizedBy: "maintainer", AuthorizedAt: fixture.authorizedAt, + ExpiresAt: fixture.authorizedAt.Add(20 * time.Minute), + Attestations: Attestations{ + SenderDomainAuthenticated: true, BounceComplaintReady: true, + PrivacyControlsReviewed: true, PilotRecipientsConfirmed: true, + }, + } +} + +func (fixture deliveryFixture) writeAuthorization(t *testing.T) string { + t.Helper() + _, raw, err := Authorize(fixture.authorizationRequest()) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(fixture.root, "launch.json") + if err := WriteAuthorization(path, raw); err != nil { + t.Fatal(err) + } + return path +} + +func (fixture deliveryFixture) writeAuthorizationAt(t *testing.T, name string, authorizedAt, expiresAt time.Time) string { + t.Helper() + request := fixture.authorizationRequest() + request.AuthorizedAt = authorizedAt + request.ExpiresAt = expiresAt + _, raw, err := Authorize(request) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(fixture.root, name) + if err := WriteAuthorization(path, raw); err != nil { + t.Fatal(err) + } + return path +} + +func (fixture deliveryFixture) openLedger(t *testing.T) *Ledger { + t.Helper() + ledger, err := OpenLedger(fixture.ledgerPath, bytes.Repeat([]byte("l"), 32)) + if err != nil { + t.Fatal(err) + } + return ledger +} + +func (fixture deliveryFixture) editionID(t *testing.T) string { + t.Helper() + plan, _, err := ReadPreflight(fixture.preflightPath) + if err != nil { + t.Fatal(err) + } + return plan.EditionID +} + +func (fixture deliveryFixture) deliveryRequest(authorizationPath string, provider Provider) DeliveryRequest { + return DeliveryRequest{ + KitDir: fixture.kitDir, SubscriberCSV: fixture.subscriberPath, SuppressionCSV: fixture.suppressionPath, + PreflightPath: fixture.preflightPath, AuthorizationPath: authorizationPath, LedgerPath: fixture.ledgerPath, + LedgerSecret: bytes.Repeat([]byte("l"), 32), SendAt: fixture.sendAt, Provider: provider, SendLive: true, + } +} diff --git a/internal/emaildelivery/ledger.go b/internal/emaildelivery/ledger.go new file mode 100644 index 0000000..a2f0992 --- /dev/null +++ b/internal/emaildelivery/ledger.go @@ -0,0 +1,462 @@ +package emaildelivery + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "net/mail" + "os" + "path/filepath" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +const minimumLedgerSecretBytes = 32 + +var ( + ErrDeliveryPending = errors.New("delivery has an unresolved pending attempt") + ErrNewAuthorizationRequired = errors.New("a reconciled non-accepted delivery requires a new launch authorization") +) + +type Ledger struct { + db *sql.DB + secret []byte +} + +type DeliveryAttempt struct { + EditionID string + Audience string + Email string + Provider string + ContentSHA256 string + AuthorizationSHA256 string + AttemptedAt time.Time +} + +type PrepareResult struct { + DeliveryKey string + IdempotencyKey string + AlreadyAccepted bool +} + +type ReconciliationRequest struct { + EditionID string + Audience string + Email string + Outcome string + ProviderMessageID string + ResolvedBy string + Evidence string + ResolvedAt time.Time +} + +type ReconciliationResult struct { + Changed bool + AlreadyResolved bool +} + +func OpenLedger(databasePath string, secret []byte) (*Ledger, error) { + databasePath = strings.TrimSpace(databasePath) + if databasePath == "" { + return nil, errors.New("delivery ledger path is required") + } + if len(secret) < minimumLedgerSecretBytes { + return nil, fmt.Errorf("delivery ledger secret must contain at least %d bytes", minimumLedgerSecretBytes) + } + absolute, err := filepath.Abs(databasePath) + if err != nil { + return nil, fmt.Errorf("resolve delivery ledger: %w", err) + } + if err := os.MkdirAll(filepath.Dir(absolute), 0o700); err != nil { + return nil, fmt.Errorf("create delivery ledger directory: %w", err) + } + db, err := sql.Open("sqlite", absolute) + if err != nil { + return nil, fmt.Errorf("open delivery ledger: %w", err) + } + db.SetMaxOpenConns(1) + ledger := &Ledger{db: db, secret: append([]byte(nil), secret...)} + if err := ledger.migrate(); err != nil { + _ = db.Close() + return nil, err + } + if err := os.Chmod(absolute, 0o600); err != nil { + _ = db.Close() + return nil, fmt.Errorf("restrict delivery ledger permissions: %w", err) + } + return ledger, nil +} + +func (ledger *Ledger) Close() error { + if ledger == nil || ledger.db == nil { + return nil + } + for index := range ledger.secret { + ledger.secret[index] = 0 + } + return ledger.db.Close() +} + +func (ledger *Ledger) Prepare(ctx context.Context, attempt DeliveryAttempt) (PrepareResult, error) { + if attempt.AttemptedAt.IsZero() || attempt.EditionID == "" || attempt.Audience == "" || attempt.Provider == "" || attempt.ContentSHA256 == "" || !validSHA256(attempt.AuthorizationSHA256) { + return PrepareResult{}, errors.New("delivery attempt is missing required fields") + } + address, err := canonicalRecipient(attempt.Email) + if err != nil { + return PrepareResult{}, err + } + deliveryKey := ledger.key("delivery", attempt.EditionID, attempt.Audience, address) + recipientKey := ledger.key("recipient", address) + idempotencyKey := "tradegravity/" + sanitizeKeyPart(attempt.EditionID) + "/" + deliveryKey[:32] + tx, err := ledger.db.BeginTx(ctx, nil) + if err != nil { + return PrepareResult{}, fmt.Errorf("begin delivery ledger transaction: %w", err) + } + defer tx.Rollback() + var status, storedContent, storedProvider, storedIdempotency, storedAuthorization, resolution, resolvedAt string + err = tx.QueryRowContext(ctx, ` + SELECT status, content_sha256, provider, idempotency_key, + authorization_sha256, COALESCE(resolution, ''), COALESCE(resolved_at, '') + FROM deliveries WHERE delivery_key = ? + `, deliveryKey).Scan(&status, &storedContent, &storedProvider, &storedIdempotency, &storedAuthorization, &resolution, &resolvedAt) + switch { + case errors.Is(err, sql.ErrNoRows): + _, err = tx.ExecContext(ctx, ` + INSERT INTO deliveries ( + delivery_key, edition_id, audience, recipient_key, provider, + content_sha256, idempotency_key, authorization_sha256, status, attempted_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?) + `, deliveryKey, attempt.EditionID, attempt.Audience, recipientKey, attempt.Provider, attempt.ContentSHA256, idempotencyKey, attempt.AuthorizationSHA256, attempt.AttemptedAt.UTC().Format(time.RFC3339)) + if err != nil { + return PrepareResult{}, fmt.Errorf("insert delivery ledger attempt: %w", err) + } + case err != nil: + return PrepareResult{}, fmt.Errorf("read delivery ledger attempt: %w", err) + case storedContent != attempt.ContentSHA256 || storedProvider != attempt.Provider || storedIdempotency != idempotencyKey: + return PrepareResult{}, errors.New("existing delivery ledger entry does not match the current content or provider") + case status == "accepted": + if err := tx.Commit(); err != nil { + return PrepareResult{}, err + } + return PrepareResult{DeliveryKey: deliveryKey, IdempotencyKey: idempotencyKey, AlreadyAccepted: true}, nil + case status == "pending" && resolution == "": + return PrepareResult{}, ErrDeliveryPending + case status == "pending" && resolution == "not_accepted" && storedAuthorization == attempt.AuthorizationSHA256: + return PrepareResult{}, ErrNewAuthorizationRequired + case status == "pending" && resolution == "not_accepted": + resolvedTime, parseErr := time.Parse(time.RFC3339, resolvedAt) + if parseErr != nil || !attempt.AttemptedAt.UTC().After(resolvedTime) { + return PrepareResult{}, errors.New("reconciled delivery retry must occur after the recorded resolution") + } + update, updateErr := tx.ExecContext(ctx, ` + UPDATE deliveries + SET authorization_sha256 = ?, attempted_at = ?, resolution = NULL, + resolved_at = NULL, resolved_by = NULL, resolution_evidence = NULL + WHERE delivery_key = ? AND status = 'pending' AND resolution = 'not_accepted' + `, attempt.AuthorizationSHA256, attempt.AttemptedAt.UTC().Format(time.RFC3339), deliveryKey) + if updateErr != nil { + return PrepareResult{}, fmt.Errorf("prepare reconciled delivery retry: %w", updateErr) + } + changed, updateErr := update.RowsAffected() + if updateErr != nil || changed != 1 { + return PrepareResult{}, errors.New("reconciled delivery retry did not update one pending entry") + } + default: + return PrepareResult{}, fmt.Errorf("delivery ledger contains unsupported status %q or resolution %q", status, resolution) + } + if err := tx.Commit(); err != nil { + return PrepareResult{}, fmt.Errorf("commit delivery ledger attempt: %w", err) + } + return PrepareResult{DeliveryKey: deliveryKey, IdempotencyKey: idempotencyKey}, nil +} + +func (ledger *Ledger) Reconcile(ctx context.Context, request ReconciliationRequest) (ReconciliationResult, error) { + if request.EditionID == "" || request.Audience == "" || request.ResolvedAt.IsZero() { + return ReconciliationResult{}, errors.New("reconciliation is missing edition, audience, or time") + } + address, err := canonicalRecipient(request.Email) + if err != nil { + return ReconciliationResult{}, err + } + if request.Outcome != "accepted" && request.Outcome != "not_accepted" { + return ReconciliationResult{}, errors.New("reconciliation outcome must be accepted or not_accepted") + } + if err := validateAuditLabel(request.ResolvedBy, "resolved-by", 120); err != nil { + return ReconciliationResult{}, err + } + if err := validateAuditLabel(request.Evidence, "evidence", 256); err != nil { + return ReconciliationResult{}, err + } + providerMessageID := strings.TrimSpace(request.ProviderMessageID) + if request.Outcome == "accepted" { + if !validProviderMessageID(providerMessageID) { + return ReconciliationResult{}, errors.New("accepted reconciliation requires a valid provider message ID") + } + } else if providerMessageID != "" { + return ReconciliationResult{}, errors.New("not_accepted reconciliation must not include a provider message ID") + } + deliveryKey := ledger.key("delivery", request.EditionID, request.Audience, address) + tx, err := ledger.db.BeginTx(ctx, nil) + if err != nil { + return ReconciliationResult{}, fmt.Errorf("begin delivery reconciliation: %w", err) + } + defer tx.Rollback() + var status, resolution, storedMessageID, attemptedAt string + err = tx.QueryRowContext(ctx, ` + SELECT status, COALESCE(resolution, ''), COALESCE(provider_message_id, ''), attempted_at + FROM deliveries WHERE delivery_key = ? + `, deliveryKey).Scan(&status, &resolution, &storedMessageID, &attemptedAt) + if errors.Is(err, sql.ErrNoRows) { + return ReconciliationResult{}, errors.New("no delivery ledger entry matches the private recipient identity") + } + if err != nil { + return ReconciliationResult{}, fmt.Errorf("read pending delivery: %w", err) + } + attemptedTime, err := time.Parse(time.RFC3339, attemptedAt) + if err != nil || request.ResolvedAt.UTC().Before(attemptedTime) { + return ReconciliationResult{}, errors.New("reconciliation time cannot predate the delivery attempt") + } + if status == "accepted" { + if request.Outcome == "accepted" && providerMessageID == storedMessageID { + if err := tx.Commit(); err != nil { + return ReconciliationResult{}, err + } + return ReconciliationResult{AlreadyResolved: true}, nil + } + return ReconciliationResult{}, errors.New("an accepted delivery cannot be reconciled as not accepted or to a different provider message") + } + if status != "pending" { + return ReconciliationResult{}, fmt.Errorf("delivery ledger contains unsupported status %q", status) + } + if resolution != "" { + if resolution == request.Outcome { + if err := tx.Commit(); err != nil { + return ReconciliationResult{}, err + } + return ReconciliationResult{AlreadyResolved: true}, nil + } + return ReconciliationResult{}, errors.New("pending delivery already has a different reconciliation outcome") + } + resolvedAt := request.ResolvedAt.UTC().Format(time.RFC3339) + var update sql.Result + if request.Outcome == "accepted" { + update, err = tx.ExecContext(ctx, ` + UPDATE deliveries + SET status = 'accepted', provider_message_id = ?, accepted_at = ?, + resolution = 'accepted', resolved_at = ?, resolved_by = ?, resolution_evidence = ? + WHERE delivery_key = ? AND status = 'pending' AND resolution IS NULL + `, providerMessageID, resolvedAt, resolvedAt, request.ResolvedBy, request.Evidence, deliveryKey) + } else { + update, err = tx.ExecContext(ctx, ` + UPDATE deliveries + SET resolution = 'not_accepted', resolved_at = ?, resolved_by = ?, resolution_evidence = ? + WHERE delivery_key = ? AND status = 'pending' AND resolution IS NULL + `, resolvedAt, request.ResolvedBy, request.Evidence, deliveryKey) + } + if err != nil { + return ReconciliationResult{}, fmt.Errorf("record delivery reconciliation: %w", err) + } + changed, err := update.RowsAffected() + if err != nil || changed != 1 { + return ReconciliationResult{}, errors.New("delivery reconciliation did not update one pending entry") + } + if err := tx.Commit(); err != nil { + return ReconciliationResult{}, fmt.Errorf("commit delivery reconciliation: %w", err) + } + return ReconciliationResult{Changed: true}, nil +} + +func (ledger *Ledger) MarkAccepted(ctx context.Context, deliveryKey, providerMessageID string, acceptedAt time.Time) error { + providerMessageID = strings.TrimSpace(providerMessageID) + if deliveryKey == "" || !validProviderMessageID(providerMessageID) || acceptedAt.IsZero() { + return errors.New("accepted delivery record is invalid") + } + result, err := ledger.db.ExecContext(ctx, ` + UPDATE deliveries + SET status = 'accepted', provider_message_id = ?, accepted_at = ? + WHERE delivery_key = ? AND status = 'pending' + `, providerMessageID, acceptedAt.UTC().Format(time.RFC3339), deliveryKey) + if err != nil { + return fmt.Errorf("mark delivery accepted: %w", err) + } + rows, err := result.RowsAffected() + if err != nil || rows != 1 { + return errors.New("delivery ledger did not contain one pending attempt") + } + return nil +} + +func (ledger *Ledger) Counts(ctx context.Context, editionID, audience string) (pending, accepted int, err error) { + rows, err := ledger.db.QueryContext(ctx, ` + SELECT status, COUNT(*) FROM deliveries + WHERE edition_id = ? AND audience = ? GROUP BY status + `, editionID, audience) + if err != nil { + return 0, 0, err + } + defer rows.Close() + for rows.Next() { + var status string + var count int + if err := rows.Scan(&status, &count); err != nil { + return 0, 0, err + } + switch status { + case "pending": + pending = count + case "accepted": + accepted = count + default: + return 0, 0, fmt.Errorf("delivery ledger contains unsupported status %q", status) + } + } + return pending, accepted, rows.Err() +} + +func (ledger *Ledger) key(purpose string, values ...string) string { + mac := hmac.New(sha256.New, ledger.secret) + _, _ = mac.Write([]byte(purpose)) + for _, value := range values { + _, _ = mac.Write([]byte{0}) + _, _ = mac.Write([]byte(value)) + } + return hex.EncodeToString(mac.Sum(nil)) +} + +func (ledger *Ledger) migrate() error { + statements := []string{ + `PRAGMA foreign_keys = ON;`, + `PRAGMA journal_mode = WAL;`, + `PRAGMA busy_timeout = 5000;`, + `CREATE TABLE IF NOT EXISTS deliveries ( + delivery_key TEXT PRIMARY KEY, + edition_id TEXT NOT NULL, + audience TEXT NOT NULL, + recipient_key TEXT NOT NULL, + provider TEXT NOT NULL, + content_sha256 TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + authorization_sha256 TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL CHECK (status IN ('pending', 'accepted')), + attempted_at TEXT NOT NULL, + accepted_at TEXT, + provider_message_id TEXT, + resolution TEXT CHECK (resolution IN ('accepted', 'not_accepted')), + resolved_at TEXT, + resolved_by TEXT, + resolution_evidence TEXT, + UNIQUE(edition_id, audience, recipient_key), + CHECK ((status = 'pending' AND accepted_at IS NULL AND provider_message_id IS NULL) + OR (status = 'accepted' AND accepted_at IS NOT NULL AND provider_message_id IS NOT NULL)) + );`, + `CREATE INDEX IF NOT EXISTS idx_deliveries_edition_status ON deliveries(edition_id, audience, status);`, + } + for _, statement := range statements { + if _, err := ledger.db.Exec(statement); err != nil { + return fmt.Errorf("migrate delivery ledger: %w", err) + } + } + columns := []struct { + name string + definition string + }{ + {"authorization_sha256", "TEXT NOT NULL DEFAULT ''"}, + {"resolution", "TEXT"}, + {"resolved_at", "TEXT"}, + {"resolved_by", "TEXT"}, + {"resolution_evidence", "TEXT"}, + } + for _, column := range columns { + if err := ledger.ensureColumn(column.name, column.definition); err != nil { + return err + } + } + return nil +} + +func (ledger *Ledger) ensureColumn(name, definition string) error { + rows, err := ledger.db.Query(`PRAGMA table_info(deliveries)`) + if err != nil { + return fmt.Errorf("inspect delivery ledger columns: %w", err) + } + found := false + for rows.Next() { + var cid int + var columnName, columnType string + var notNull, primaryKey int + var defaultValue any + if err := rows.Scan(&cid, &columnName, &columnType, ¬Null, &defaultValue, &primaryKey); err != nil { + _ = rows.Close() + return err + } + if columnName == name { + found = true + } + } + if err := rows.Close(); err != nil { + return err + } + if found { + return nil + } + if _, err := ledger.db.Exec("ALTER TABLE deliveries ADD COLUMN " + name + " " + definition); err != nil { + return fmt.Errorf("add delivery ledger column %s: %w", name, err) + } + return nil +} + +func validSHA256(value string) bool { + if len(value) != 64 { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +func validateAuditLabel(value, name string, limit int) error { + trimmed := strings.TrimSpace(value) + if value != trimmed || trimmed == "" || len([]rune(trimmed)) > limit || strings.Contains(trimmed, "@") || strings.Contains(trimmed, "://") { + return fmt.Errorf("%s must be a non-sensitive bounded label", name) + } + for _, char := range value { + if char < 0x20 || char == 0x7f { + return fmt.Errorf("%s contains a control character", name) + } + } + return nil +} + +func validProviderMessageID(value string) bool { + return value != "" && len(value) <= 256 && !strings.ContainsAny(value, "\r\n@") && !strings.Contains(value, "://") +} + +func canonicalRecipient(value string) (string, error) { + value = strings.TrimSpace(value) + parsed, err := mail.ParseAddress(value) + if err != nil || parsed.Address != value || strings.Count(value, "@") != 1 || strings.ContainsAny(value, "\r\n") { + return "", errors.New("recipient must be one plain email addr-spec") + } + return strings.ToLower(value), nil +} + +func sanitizeKeyPart(value string) string { + var result strings.Builder + for _, char := range value { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || (char >= '0' && char <= '9') || char == '-' || char == '_' { + result.WriteRune(char) + } + } + if result.Len() == 0 { + return "edition" + } + if result.Len() > 80 { + return result.String()[:80] + } + return result.String() +} diff --git a/internal/emaildelivery/resend.go b/internal/emaildelivery/resend.go new file mode 100644 index 0000000..8b90806 --- /dev/null +++ b/internal/emaildelivery/resend.go @@ -0,0 +1,107 @@ +package emaildelivery + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +const ( + resendEndpoint = "https://api.resend.com/emails" + maximumProviderBody = 64 << 10 +) + +type ResendProvider struct { + apiKey string + client *http.Client + endpoint string +} + +func NewResendProvider(apiKey string, client *http.Client) (*ResendProvider, error) { + apiKey = strings.TrimSpace(apiKey) + if apiKey == "" || strings.ContainsAny(apiKey, "\r\n") { + return nil, errors.New("RESEND_API_KEY is required") + } + if client == nil { + client = &http.Client{Timeout: 15 * time.Second} + } + return &ResendProvider{apiKey: apiKey, client: client, endpoint: resendEndpoint}, nil +} + +func (provider *ResendProvider) Name() string { return "resend" } + +func (provider *ResendProvider) Send(ctx context.Context, message Message, idempotencyKey string) (string, error) { + return provider.sendTo(ctx, message, idempotencyKey, provider.endpoint) +} + +func (provider *ResendProvider) sendTo(ctx context.Context, message Message, idempotencyKey, endpoint string) (string, error) { + if strings.TrimSpace(idempotencyKey) == "" || len(idempotencyKey) > 256 || strings.ContainsAny(idempotencyKey, "\r\n") { + return "", errors.New("provider idempotency key is invalid") + } + parsedEndpoint, err := url.Parse(endpoint) + if err != nil || parsedEndpoint.Scheme != "https" || parsedEndpoint.Host == "" || parsedEndpoint.User != nil || parsedEndpoint.RawQuery != "" || parsedEndpoint.Fragment != "" { + return "", errors.New("provider endpoint must be absolute HTTPS") + } + payload := struct { + From string `json:"from"` + To []string `json:"to"` + Subject string `json:"subject"` + HTML string `json:"html"` + Text string `json:"text"` + ReplyTo string `json:"reply_to,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + }{ + From: message.From, To: []string{message.To}, Subject: message.Subject, + HTML: message.HTML, Text: message.Text, ReplyTo: message.ReplyTo, + } + if message.ListUnsubscribe != "" || message.ListUnsubscribePost != "" { + if message.ListUnsubscribe == "" || message.ListUnsubscribePost == "" { + return "", errors.New("unsubscribe headers must be supplied together") + } + payload.Headers = map[string]string{ + "List-Unsubscribe": message.ListUnsubscribe, + "List-Unsubscribe-Post": message.ListUnsubscribePost, + } + } + raw, err := json.Marshal(payload) + if err != nil { + return "", fmt.Errorf("encode provider request: %w", err) + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(raw)) + if err != nil { + return "", fmt.Errorf("build provider request: %w", err) + } + request.Header.Set("Authorization", "Bearer "+provider.apiKey) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", idempotencyKey) + request.Header.Set("User-Agent", "TradeGravity-email-pilot/1.0") + response, err := provider.client.Do(request) + if err != nil { + return "", errors.New("provider request failed without a confirmed response") + } + defer response.Body.Close() + body, err := io.ReadAll(io.LimitReader(response.Body, maximumProviderBody+1)) + if err != nil { + return "", errors.New("provider response could not be read") + } + if len(body) > maximumProviderBody { + return "", errors.New("provider response exceeded the safety limit") + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return "", fmt.Errorf("provider rejected the request with HTTP %d", response.StatusCode) + } + var result struct { + ID string `json:"id"` + } + if err := decodeStrictJSON(body, &result); err != nil || strings.TrimSpace(result.ID) == "" || len(result.ID) > 256 || strings.ContainsAny(result.ID, "\r\n") { + return "", errors.New("provider success response omitted a valid message ID") + } + return result.ID, nil +} diff --git a/internal/socialpreflight/preflight.go b/internal/socialpreflight/preflight.go new file mode 100644 index 0000000..d0363a7 --- /dev/null +++ b/internal/socialpreflight/preflight.go @@ -0,0 +1,253 @@ +package socialpreflight + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "image/png" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + "time" + "unicode/utf8" + + "tradegravity/internal/distributionkit" +) + +const ( + schemaVersion = "1.0" + toolVersion = "tradegravity-instagram-preflight/1.0" + maximumTextFileSize = 64 << 10 + maximumPNGFileSize = 20 << 20 + captionRuneLimit = 1800 +) + +var hashtagPattern = regexp.MustCompile(`^#[A-Za-z0-9]+$`) + +type Plan struct { + SchemaVersion string `json:"schema_version"` + Tool string `json:"tool"` + Status string `json:"status"` + Channel string `json:"channel"` + EditionID string `json:"edition_id"` + Theme string `json:"theme"` + GeneratedAt string `json:"generated_at"` + ManifestSHA256 string `json:"manifest_sha256"` + ApprovalSHA256 string `json:"approval_sha256"` + SlideCount int `json:"slide_count"` + Width int `json:"width"` + Height int `json:"height"` + CaptionRunes int `json:"caption_runes"` + HashtagCount int `json:"hashtag_count"` + AltTextSections int `json:"alt_text_sections"` + Checks Checks `json:"checks"` + ContainsCaptionText bool `json:"contains_caption_text"` + ContainsCredentials bool `json:"contains_credentials"` + ManualUploadRequired bool `json:"manual_upload_required"` + AutomaticPublishAuthorized bool `json:"automatic_publish_authorized"` +} + +type Checks struct { + ContentApproved bool `json:"content_approved"` + ManifestIntegrity bool `json:"manifest_integrity"` + PNGDimensions bool `json:"png_dimensions"` + CaptionEvidenceAndScope bool `json:"caption_evidence_and_scope"` + AltTextComplete bool `json:"alt_text_complete"` +} + +type Result struct { + Plan Plan + JSON []byte +} + +func Build(kitDir string, generatedAt time.Time) (Result, error) { + if generatedAt.IsZero() { + return Result{}, errors.New("Instagram preflight generation time is required") + } + root, err := canonicalDirectory(kitDir) + if err != nil { + return Result{}, err + } + approval, manifest, err := distributionkit.VerifyApproved(root, "instagram") + if err != nil { + return Result{}, fmt.Errorf("verify approved Instagram kit: %w", err) + } + caption, err := readBoundedRegular(filepath.Join(root, filepath.FromSlash(manifest.Carousel.CaptionPath)), "caption") + if err != nil { + return Result{}, err + } + captionRunes, hashtags, err := validateCaption(caption, manifest.BaseURL) + if err != nil { + return Result{}, err + } + altText, err := readBoundedRegular(filepath.Join(root, "carousel", "alt-text.md"), "alt text") + if err != nil { + return Result{}, err + } + sections, err := validateAltText(altText, manifest.Carousel.SlideCount) + if err != nil { + return Result{}, err + } + for index := 1; index <= manifest.Carousel.SlideCount; index++ { + path := filepath.Join(root, "carousel", fmt.Sprintf("slide-%02d.png", index)) + raw, readErr := readBoundedRegular(path, "carousel PNG") + if readErr != nil { + return Result{}, readErr + } + configuration, configErr := png.DecodeConfig(bytes.NewReader(raw)) + if configErr != nil { + return Result{}, fmt.Errorf("inspect carousel PNG %d: %w", index, configErr) + } + if configuration.Width != manifest.Carousel.Width || configuration.Height != manifest.Carousel.Height { + return Result{}, fmt.Errorf("carousel PNG %d dimensions do not match the manifest", index) + } + image, decodeErr := png.Decode(bytes.NewReader(raw)) + if decodeErr != nil { + return Result{}, fmt.Errorf("decode carousel PNG %d: %w", index, decodeErr) + } + if image.Bounds().Dx() != configuration.Width || image.Bounds().Dy() != configuration.Height { + return Result{}, fmt.Errorf("carousel PNG %d dimensions do not match the manifest", index) + } + } + approvalRaw, err := readBoundedRegular(filepath.Join(root, "approval.json"), "approval") + if err != nil { + return Result{}, err + } + approvalDigest := sha256.Sum256(approvalRaw) + plan := Plan{ + SchemaVersion: schemaVersion, Tool: toolVersion, Status: "ready_for_manual_preview", Channel: "instagram", + EditionID: manifest.EditionID, Theme: manifest.Carousel.Theme, GeneratedAt: generatedAt.UTC().Format(time.RFC3339), + ManifestSHA256: approval.ManifestSHA256, ApprovalSHA256: hex.EncodeToString(approvalDigest[:]), + SlideCount: manifest.Carousel.SlideCount, Width: manifest.Carousel.Width, Height: manifest.Carousel.Height, + CaptionRunes: captionRunes, HashtagCount: hashtags, AltTextSections: sections, + Checks: Checks{ContentApproved: true, ManifestIntegrity: true, PNGDimensions: true, CaptionEvidenceAndScope: true, AltTextComplete: true}, + ContainsCaptionText: false, ContainsCredentials: false, ManualUploadRequired: true, AutomaticPublishAuthorized: false, + } + raw, err := json.MarshalIndent(plan, "", " ") + if err != nil { + return Result{}, fmt.Errorf("encode Instagram preflight: %w", err) + } + return Result{Plan: plan, JSON: append(raw, '\n')}, nil +} + +func validateCaption(raw []byte, baseURL string) (int, int, error) { + if !utf8.Valid(raw) { + return 0, 0, errors.New("Instagram caption is not valid UTF-8") + } + value := string(raw) + runes := len([]rune(value)) + if runes == 0 || runes > captionRuneLimit { + return 0, 0, fmt.Errorf("Instagram caption must contain 1-%d runes", captionRuneLimit) + } + if strings.Contains(value, "{{") || strings.ContainsRune(value, '\x00') { + return 0, 0, errors.New("Instagram caption contains an unresolved or invalid value") + } + if !strings.Contains(value, strings.TrimSpace(baseURL)) || !strings.Contains(value, "Scope note:") || !strings.Contains(value, "not a physical shipment route") { + return 0, 0, errors.New("Instagram caption must retain the evidence URL and scope note") + } + seen := map[string]bool{} + count := 0 + for _, field := range strings.Fields(value) { + if !strings.HasPrefix(field, "#") { + continue + } + if !hashtagPattern.MatchString(field) { + return 0, 0, errors.New("Instagram caption contains an invalid hashtag") + } + key := strings.ToLower(field) + if seen[key] { + return 0, 0, errors.New("Instagram caption contains a duplicate hashtag") + } + seen[key] = true + count++ + } + if count < 1 || count > 8 { + return 0, 0, errors.New("Instagram caption must contain 1-8 restrained hashtags") + } + return runes, count, nil +} + +func validateAltText(raw []byte, want int) (int, error) { + if !utf8.Valid(raw) || strings.Contains(string(raw), "{{") { + return 0, errors.New("carousel alt text is invalid") + } + sections := strings.Count(string(raw), "\n## Slide ") + evidence := strings.Count(string(raw), "\nEvidence:\n") + if sections != want || evidence != want { + return 0, fmt.Errorf("carousel alt text must contain %d complete slide sections", want) + } + return sections, nil +} + +func readBoundedRegular(path, label string) ([]byte, error) { + info, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("inspect %s: %w", label, err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("%s must be a regular file", label) + } + if (filepath.Ext(path) == ".png" && info.Size() > maximumPNGFileSize) || (filepath.Ext(path) != ".png" && info.Size() > maximumTextFileSize) { + return nil, fmt.Errorf("%s exceeds the safety limit", label) + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", label, err) + } + return raw, nil +} + +func Write(path, kitDir string, raw []byte) error { + target, err := filepath.Abs(strings.TrimSpace(path)) + if err != nil || strings.TrimSpace(path) == "" { + return errors.New("Instagram preflight output path is required") + } + root, err := canonicalDirectory(kitDir) + if err != nil { + return err + } + rel, err := filepath.Rel(root, target) + if err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return errors.New("Instagram preflight output must remain outside the distribution kit") + } + if filepath.Ext(target) != ".json" { + return errors.New("Instagram preflight output must use a .json extension") + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + file, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + if errors.Is(err, fs.ErrExist) { + return errors.New("Instagram preflight output already exists") + } + return err + } + if _, err = file.Write(raw); err != nil { + _ = file.Close() + _ = os.Remove(target) + return err + } + if err = file.Close(); err != nil { + _ = os.Remove(target) + return err + } + return nil +} + +func canonicalDirectory(value string) (string, error) { + absolute, err := filepath.Abs(strings.TrimSpace(value)) + if err != nil || strings.TrimSpace(value) == "" { + return "", errors.New("distribution-kit directory is required") + } + info, err := os.Stat(absolute) + if err != nil || !info.IsDir() { + return "", errors.New("distribution-kit directory is unavailable") + } + return absolute, nil +} diff --git a/internal/socialpreflight/preflight_test.go b/internal/socialpreflight/preflight_test.go new file mode 100644 index 0000000..a9bc921 --- /dev/null +++ b/internal/socialpreflight/preflight_test.go @@ -0,0 +1,115 @@ +package socialpreflight + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "tradegravity/internal/distributionkit" +) + +func TestBuildValidatesApprovedManualInstagramPackageWithoutContentLeak(t *testing.T) { + kit := approvedKit(t, []string{"instagram"}) + result, err := Build(kit, time.Date(2026, 7, 17, 14, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal(err) + } + plan := result.Plan + if plan.Status != "ready_for_manual_preview" || plan.Channel != "instagram" || plan.Theme != distributionkit.ThemeEditorialLight || plan.SlideCount != 6 || plan.Width != 1080 || plan.Height != 1350 { + t.Fatalf("unexpected plan identity: %+v", plan) + } + if !plan.Checks.ContentApproved || !plan.Checks.ManifestIntegrity || !plan.Checks.PNGDimensions || !plan.Checks.CaptionEvidenceAndScope || !plan.Checks.AltTextComplete { + t.Fatalf("preflight checks are incomplete: %+v", plan.Checks) + } + if plan.CaptionRunes < 100 || plan.CaptionRunes > captionRuneLimit || plan.HashtagCount != 4 || plan.AltTextSections != 6 { + t.Fatalf("unexpected content aggregates: %+v", plan) + } + if !plan.ManualUploadRequired || plan.AutomaticPublishAuthorized || plan.ContainsCaptionText || plan.ContainsCredentials { + t.Fatalf("unsafe publication flags: %+v", plan) + } + if bytes.Contains(result.JSON, []byte("Korea")) || bytes.Contains(result.JSON, []byte("example.org")) || bytes.Contains(result.JSON, []byte("#TradeGravity")) { + t.Fatal("aggregate preflight leaked caption text, evidence URL, or hashtags") + } + var decoded Plan + if err := json.Unmarshal(result.JSON, &decoded); err != nil || decoded.EditionID != plan.EditionID { + t.Fatalf("preflight JSON is invalid: %v", err) + } +} + +func TestBuildRequiresInstagramApprovalAndRejectsTampering(t *testing.T) { + if _, err := Build(approvedKit(t, []string{"email"}), time.Now()); err == nil || !strings.Contains(err.Error(), "not approved") { + t.Fatalf("email-only approval error = %v", err) + } + kit := approvedKit(t, []string{"instagram"}) + caption := filepath.Join(kit, "carousel", "caption.md") + if err := os.WriteFile(caption, []byte("replacement\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Build(kit, time.Now()); err == nil || (!strings.Contains(err.Error(), "size changed") && !strings.Contains(err.Error(), "digest changed")) { + t.Fatalf("tampered caption error = %v", err) + } +} + +func TestCaptionAndAltTextContractsFailClosed(t *testing.T) { + valid := []byte("Title\n\nEvidence https://example.org/TradeGravity/\nScope note: descriptive evidence; not a physical shipment route.\n#TradeGravity\n") + if _, count, err := validateCaption(valid, "https://example.org/TradeGravity/"); err != nil || count != 1 { + t.Fatalf("valid caption = count %d err %v", count, err) + } + for name, raw := range map[string][]byte{ + "missing evidence": []byte("Scope note: not a physical shipment route. #TradeGravity"), + "missing scope": []byte("https://example.org/TradeGravity/ #TradeGravity"), + "duplicate tag": []byte("https://example.org/TradeGravity/ Scope note: not a physical shipment route. #TradeGravity #tradegravity"), + "placeholder": []byte("https://example.org/TradeGravity/ Scope note: not a physical shipment route. {{value}} #TradeGravity"), + } { + t.Run(name, func(t *testing.T) { + if _, _, err := validateCaption(raw, "https://example.org/TradeGravity/"); err == nil { + t.Fatal("invalid caption was accepted") + } + }) + } + if _, err := validateAltText([]byte("# Carousel alt text\n\n## Slide 1\n\nEvidence:\n- source\n"), 6); err == nil { + t.Fatal("incomplete alt text was accepted") + } +} + +func TestWriteRefusesKitPathAndOverwrite(t *testing.T) { + kit := approvedKit(t, []string{"instagram"}) + if err := Write(filepath.Join(kit, "preflight.json"), kit, []byte("{}\n")); err == nil { + t.Fatal("Write() placed preflight inside the approved kit") + } + out := filepath.Join(t.TempDir(), "instagram-preflight.json") + if err := Write(out, kit, []byte("{}\n")); err != nil { + t.Fatal(err) + } + if err := Write(out, kit, []byte("changed\n")); err == nil { + t.Fatal("Write() overwrote an existing preflight") + } +} + +func approvedKit(t *testing.T, channels []string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "examples", "sample-data", "briefing.json")) + if err != nil { + t.Fatal(err) + } + bundle, err := distributionkit.BuildWithOptions(raw, "https://example.org/TradeGravity/", distributionkit.BuildOptions{Theme: distributionkit.ThemeEditorialLight}) + if err != nil { + t.Fatal(err) + } + kit := filepath.Join(t.TempDir(), "kit") + if err := distributionkit.Write(kit, bundle); err != nil { + t.Fatal(err) + } + _, approvalRaw, err := distributionkit.Approve(kit, distributionkit.ApprovalRequest{Reviewer: "reviewer", Audience: "social-pilot", Channels: channels, ApprovedAt: time.Date(2026, 7, 17, 13, 0, 0, 0, time.UTC), Attested: true}) + if err != nil { + t.Fatal(err) + } + if err := distributionkit.WriteApproval(kit, approvalRaw); err != nil { + t.Fatal(err) + } + return kit +} diff --git a/internal/subscriptions/http.go b/internal/subscriptions/http.go new file mode 100644 index 0000000..166e9f3 --- /dev/null +++ b/internal/subscriptions/http.go @@ -0,0 +1,293 @@ +package subscriptions + +import ( + "context" + "encoding/json" + "errors" + "html/template" + "io" + "net/http" + "strings" + "time" + + svix "github.com/svix/svix-webhooks/go" +) + +const maximumWebhookBody = 64 << 10 + +var unsubscribePage = template.Must(template.New("unsubscribe").Parse(` +TradeGravity email preferences +

{{.Title}}

{{.Message}}

{{if .Token}}
{{end}}
`)) + +var signupPage = template.Must(template.New("signup").Parse(` +TradeGravity email briefing +

{{.Title}}

{{.Message}}

{{if .Form}}
privacy notice.
{{end}}
`)) + +var confirmationPage = template.Must(template.New("confirmation").Parse(` +TradeGravity email confirmation +

{{.Title}}

{{.Message}}

{{if .Token}}
{{end}}
`)) + +type unsubscribeView struct { + Title string + Message string + Token string + Action string +} + +type signupView struct { + Title, Message, Action, PrivacyURL string + Form bool +} + +type ConfirmationEmail struct { + To, ConfirmationURL, IdempotencyKey string + ExpiresAt time.Time +} + +type ConfirmationSender interface { + SendConfirmation(context.Context, ConfirmationEmail) (string, error) +} + +type SignupOptions struct { + Config SignupConfig + Sender ConfirmationSender +} + +type HandlerOptions struct { + Now func() time.Time + ResendWebhookSecret string + Signup *SignupOptions +} + +func (registry *Registry) Handler(now func() time.Time) http.Handler { + handler, _ := registry.HandlerWithOptions(HandlerOptions{Now: now}) + return handler +} + +func (registry *Registry) HandlerWithResendWebhook(webhookSecret string, now func() time.Time) (http.Handler, error) { + return registry.HandlerWithOptions(HandlerOptions{Now: now, ResendWebhookSecret: webhookSecret}) +} + +func (registry *Registry) HandlerWithOptions(options HandlerOptions) (http.Handler, error) { + now := options.Now + if now == nil { + now = time.Now + } + var webhook *svix.Webhook + if strings.TrimSpace(options.ResendWebhookSecret) != "" { + var err error + webhook, err = svix.NewWebhook(strings.TrimSpace(options.ResendWebhookSecret)) + if err != nil { + return nil, errors.New("RESEND_WEBHOOK_SECRET is invalid") + } + } + if options.Signup != nil { + if options.Signup.Sender == nil { + return nil, errors.New("confirmation sender is required") + } + if err := validateSignupConfig(options.Signup.Config); err != nil { + return nil, err + } + } + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(response http.ResponseWriter, request *http.Request) { + setSecurityHeaders(response) + if request.Method != http.MethodGet { + response.Header().Set("Allow", http.MethodGet) + http.Error(response, "method not allowed", http.StatusMethodNotAllowed) + return + } + ctx, cancel := context.WithTimeout(request.Context(), time.Second) + defer cancel() + if err := registry.db.PingContext(ctx); err != nil { + http.Error(response, "unavailable", http.StatusServiceUnavailable) + return + } + response.Header().Set("Content-Type", "text/plain; charset=utf-8") + _, _ = response.Write([]byte("ok\n")) + }) + unsubscribePath := registry.unsubscribeBase.Path + mux.HandleFunc(unsubscribePath, func(response http.ResponseWriter, request *http.Request) { + setSecurityHeaders(response) + token := request.URL.Query().Get("token") + switch request.Method { + case http.MethodGet: + if err := registry.ValidateToken(token); err != nil { + renderUnsubscribe(response, http.StatusBadRequest, unsubscribeView{Title: "Link unavailable", Message: "This unsubscribe link is invalid. No subscription was changed."}) + return + } + renderUnsubscribe(response, http.StatusOK, unsubscribeView{Title: "Unsubscribe from TradeGravity", Message: "Confirm that you no longer want this publication. Opening this page has not changed your subscription.", Token: token, Action: unsubscribePath}) + case http.MethodPost: + request.Body = http.MaxBytesReader(response, request.Body, 1024) + if !strings.HasPrefix(strings.ToLower(request.Header.Get("Content-Type")), "application/x-www-form-urlencoded") { + renderUnsubscribe(response, http.StatusUnsupportedMediaType, unsubscribeView{Title: "Request unavailable", Message: "The one-click unsubscribe request used an unsupported media type. No subscription was changed."}) + return + } + if err := request.ParseForm(); err != nil || request.Form.Get("List-Unsubscribe") != "One-Click" { + renderUnsubscribe(response, http.StatusBadRequest, unsubscribeView{Title: "Request unavailable", Message: "The one-click unsubscribe request was invalid. No subscription was changed."}) + return + } + _, err := registry.Unsubscribe(request.Context(), token, now().UTC()) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, ErrInvalidToken) { + status = http.StatusBadRequest + } + renderUnsubscribe(response, status, unsubscribeView{Title: "Request unavailable", Message: "The unsubscribe request could not be completed. No additional information was disclosed."}) + return + } + renderUnsubscribe(response, http.StatusOK, unsubscribeView{Title: "Unsubscribed", Message: "This address is now suppressed from the selected TradeGravity publication."}) + default: + response.Header().Set("Allow", strings.Join([]string{http.MethodGet, http.MethodPost}, ", ")) + http.Error(response, "method not allowed", http.StatusMethodNotAllowed) + } + }) + if options.Signup != nil { + prefix := strings.TrimSuffix(unsubscribePath, "unsubscribe") + subscribePath, confirmPath := prefix+"subscribe", prefix+"confirm" + config, sender := options.Signup.Config, options.Signup.Sender + mux.HandleFunc(subscribePath, func(response http.ResponseWriter, request *http.Request) { + setSecurityHeaders(response) + switch request.Method { + case http.MethodGet: + renderSignup(response, http.StatusOK, signupView{Title: "Subscribe to TradeGravity", Message: "Receive the research briefing after confirming your address.", Action: subscribePath, PrivacyURL: config.PrivacyNoticeURL, Form: true}) + case http.MethodPost: + request.Body = http.MaxBytesReader(response, request.Body, 4096) + if !strings.HasPrefix(strings.ToLower(request.Header.Get("Content-Type")), "application/x-www-form-urlencoded") { + renderSignup(response, http.StatusUnsupportedMediaType, signupView{Title: "Request unavailable", Message: "The subscription request was invalid."}) + return + } + if err := request.ParseForm(); err != nil || request.Form.Get("privacy") != "accepted" || request.Form.Get("website") != "" { + renderSignup(response, http.StatusBadRequest, signupView{Title: "Request unavailable", Message: "The subscription request was invalid."}) + return + } + dispatch, err := registry.RequestSubscription(request.Context(), request.Form.Get("email"), config, now().UTC()) + if err != nil { + renderSignup(response, http.StatusBadRequest, signupView{Title: "Request unavailable", Message: "The subscription request was invalid."}) + return + } + if dispatch.ShouldDispatch { + messageID, sendErr := sender.SendConfirmation(request.Context(), ConfirmationEmail{To: dispatch.Email, ConfirmationURL: dispatch.ConfirmationURL, IdempotencyKey: dispatch.IdempotencyKey, ExpiresAt: dispatch.ExpiresAt}) + if sendErr != nil || registry.MarkConfirmationDispatched(request.Context(), dispatch.PendingID, messageID, now().UTC()) != nil { + renderSignup(response, http.StatusServiceUnavailable, signupView{Title: "Temporarily unavailable", Message: "The confirmation request could not be completed. Please try again later."}) + return + } + } + renderSignup(response, http.StatusAccepted, signupView{Title: "Check your inbox", Message: "If this address is eligible, a confirmation email has been sent. The subscription is not active until it is confirmed."}) + default: + response.Header().Set("Allow", strings.Join([]string{http.MethodGet, http.MethodPost}, ", ")) + http.Error(response, "method not allowed", http.StatusMethodNotAllowed) + } + }) + mux.HandleFunc(confirmPath, func(response http.ResponseWriter, request *http.Request) { + setSecurityHeaders(response) + token := request.URL.Query().Get("token") + switch request.Method { + case http.MethodGet: + if registry.ValidateConfirmation(token, now().UTC()) != nil { + renderConfirmation(response, http.StatusBadRequest, "Link unavailable", "This confirmation link is invalid or expired.", "", confirmPath) + return + } + renderConfirmation(response, http.StatusOK, "Confirm TradeGravity subscription", "Opening this page has not activated the subscription. Confirm below to finish.", token, confirmPath) + case http.MethodPost: + request.Body = http.MaxBytesReader(response, request.Body, 1024) + if !strings.HasPrefix(strings.ToLower(request.Header.Get("Content-Type")), "application/x-www-form-urlencoded") { + renderConfirmation(response, http.StatusUnsupportedMediaType, "Request unavailable", "The confirmation request was invalid.", "", confirmPath) + return + } + if err := request.ParseForm(); err != nil || request.Form.Get("confirm") != "yes" { + renderConfirmation(response, http.StatusBadRequest, "Request unavailable", "The confirmation request was invalid.", "", confirmPath) + return + } + if _, err := registry.ConfirmSubscription(request.Context(), token, now().UTC()); err != nil { + renderConfirmation(response, http.StatusBadRequest, "Request unavailable", "The confirmation request could not be completed.", "", confirmPath) + return + } + renderConfirmation(response, http.StatusOK, "Subscription confirmed", "The address is now subscribed to the selected TradeGravity briefing.", "", confirmPath) + default: + response.Header().Set("Allow", strings.Join([]string{http.MethodGet, http.MethodPost}, ", ")) + http.Error(response, "method not allowed", http.StatusMethodNotAllowed) + } + }) + } + if webhook != nil { + webhookPath := strings.TrimSuffix(unsubscribePath, "unsubscribe") + "webhooks/resend" + mux.HandleFunc(webhookPath, func(response http.ResponseWriter, request *http.Request) { + setSecurityHeaders(response) + if request.Method != http.MethodPost { + response.Header().Set("Allow", http.MethodPost) + http.Error(response, "method not allowed", http.StatusMethodNotAllowed) + return + } + if !strings.HasPrefix(strings.ToLower(request.Header.Get("Content-Type")), "application/json") { + http.Error(response, "invalid webhook", http.StatusUnsupportedMediaType) + return + } + request.Body = http.MaxBytesReader(response, request.Body, maximumWebhookBody) + raw, err := io.ReadAll(request.Body) + if err != nil || webhook.Verify(raw, request.Header) != nil { + http.Error(response, "invalid webhook", http.StatusBadRequest) + return + } + var event struct { + Type string `json:"type"` + CreatedAt string `json:"created_at"` + Data struct { + To []string `json:"to"` + } `json:"data"` + } + if err := json.Unmarshal(raw, &event); err != nil { + http.Error(response, "invalid webhook", http.StatusBadRequest) + return + } + reason := "" + switch event.Type { + case "email.bounced": + reason = "bounced" + case "email.complained": + reason = "complaint" + case "email.suppressed": + reason = "invalid" + default: + response.WriteHeader(http.StatusNoContent) + return + } + occurredAt, err := time.Parse(time.RFC3339Nano, event.CreatedAt) + if err != nil || len(event.Data.To) != 1 { + http.Error(response, "invalid webhook", http.StatusBadRequest) + return + } + if _, err := registry.SuppressAddress(request.Context(), event.Data.To[0], reason, request.Header.Get("svix-id"), event.Type, occurredAt, now().UTC()); err != nil { + http.Error(response, "webhook unavailable", http.StatusInternalServerError) + return + } + response.WriteHeader(http.StatusNoContent) + }) + } + return mux, nil +} + +func renderSignup(response http.ResponseWriter, status int, view signupView) { + response.Header().Set("Content-Type", "text/html; charset=utf-8") + response.WriteHeader(status) + _ = signupPage.Execute(response, view) +} +func renderConfirmation(response http.ResponseWriter, status int, title, message, token, action string) { + response.Header().Set("Content-Type", "text/html; charset=utf-8") + response.WriteHeader(status) + _ = confirmationPage.Execute(response, unsubscribeView{Title: title, Message: message, Token: token, Action: action}) +} + +func renderUnsubscribe(response http.ResponseWriter, status int, view unsubscribeView) { + response.Header().Set("Content-Type", "text/html; charset=utf-8") + response.WriteHeader(status) + _ = unsubscribePage.Execute(response, view) +} + +func setSecurityHeaders(response http.ResponseWriter) { + response.Header().Set("Cache-Control", "no-store") + response.Header().Set("Content-Security-Policy", "default-src 'none'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'") + response.Header().Set("Referrer-Policy", "no-referrer") + response.Header().Set("X-Content-Type-Options", "nosniff") + response.Header().Set("X-Frame-Options", "DENY") +} diff --git a/internal/subscriptions/registry.go b/internal/subscriptions/registry.go new file mode 100644 index 0000000..f9ffb0f --- /dev/null +++ b/internal/subscriptions/registry.go @@ -0,0 +1,753 @@ +package subscriptions + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/base64" + "encoding/csv" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/mail" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "time" + "unicode" + + _ "modernc.org/sqlite" +) + +const ( + tokenVersion = 1 + minimumSecretBytes = 32 +) + +var consentHeader = []string{"email", "audience", "status", "consented_at", "consent_method", "consent_source", "privacy_notice_version"} +var deliveryHeader = []string{"email", "audience", "status", "consented_at", "consent_method", "consent_source", "privacy_notice_version", "unsubscribe_url"} +var suppressionHeader = []string{"email", "reason", "suppressed_at"} + +var ErrInvalidToken = errors.New("invalid unsubscribe token") + +type Registry struct { + db *sql.DB + secret []byte + unsubscribeBase *url.URL +} + +type ImportResult struct { + Inserted int + Updated int + SuppressedSkipped int +} + +type UnsubscribeResult struct { + Changed bool + AlreadyStopped bool +} + +type FeedbackResult struct { + Duplicate bool + SubscriptionsStopped int64 +} + +type consentRecord struct { + Email string + Audience string + ConsentedAt time.Time + ConsentSource string + PrivacyNoticeVersion string +} + +type storedSubscription struct { + ID string + Email string + Audience string + Status string + ConsentedAt string + ConsentSource string + PrivacyNoticeVersion string +} + +type tokenPayload struct { + Version int `json:"v"` + ID string `json:"id"` +} + +func Open(databasePath string, secret []byte, publicBaseURL string) (*Registry, error) { + databasePath = strings.TrimSpace(databasePath) + if databasePath == "" { + return nil, errors.New("subscription database path is required") + } + if len(secret) < minimumSecretBytes { + return nil, fmt.Errorf("unsubscribe secret must contain at least %d bytes", minimumSecretBytes) + } + base, err := normalizePublicBaseURL(publicBaseURL) + if err != nil { + return nil, err + } + absolute, err := filepath.Abs(databasePath) + if err != nil { + return nil, fmt.Errorf("resolve subscription database: %w", err) + } + if err := os.MkdirAll(filepath.Dir(absolute), 0o700); err != nil { + return nil, fmt.Errorf("create subscription database directory: %w", err) + } + db, err := sql.Open("sqlite", absolute) + if err != nil { + return nil, fmt.Errorf("open subscription database: %w", err) + } + db.SetMaxOpenConns(1) + registry := &Registry{db: db, secret: append([]byte(nil), secret...), unsubscribeBase: base} + if err := registry.migrate(); err != nil { + _ = db.Close() + return nil, err + } + if err := os.Chmod(absolute, 0o600); err != nil { + _ = db.Close() + return nil, fmt.Errorf("restrict subscription database permissions: %w", err) + } + return registry, nil +} + +func (registry *Registry) Close() error { + if registry == nil || registry.db == nil { + return nil + } + for index := range registry.secret { + registry.secret[index] = 0 + } + return registry.db.Close() +} + +func (registry *Registry) ImportConsents(ctx context.Context, raw []byte, importedAt time.Time) (ImportResult, error) { + if importedAt.IsZero() { + return ImportResult{}, errors.New("consent import time is required") + } + records, err := parseConsentCSV(raw, importedAt) + if err != nil { + return ImportResult{}, err + } + tx, err := registry.db.BeginTx(ctx, nil) + if err != nil { + return ImportResult{}, fmt.Errorf("begin consent import: %w", err) + } + defer tx.Rollback() + result := ImportResult{} + for _, record := range records { + var globallySuppressed int + err := tx.QueryRowContext(ctx, ` + SELECT 1 FROM address_suppressions WHERE email_normalized = ? + `, record.Email).Scan(&globallySuppressed) + if err == nil { + result.SuppressedSkipped++ + continue + } + if !errors.Is(err, sql.ErrNoRows) { + return ImportResult{}, fmt.Errorf("check address suppression: %w", err) + } + var existingID, existingStatus, existingConsentedAt string + err = tx.QueryRowContext(ctx, ` + SELECT id, status, consented_at + FROM subscriptions + WHERE email_normalized = ? AND audience = ? + `, record.Email, record.Audience).Scan(&existingID, &existingStatus, &existingConsentedAt) + switch { + case errors.Is(err, sql.ErrNoRows): + id, idErr := randomID() + if idErr != nil { + return ImportResult{}, idErr + } + _, err = tx.ExecContext(ctx, ` + INSERT INTO subscriptions ( + id, email, email_normalized, audience, status, consented_at, + consent_method, consent_source, privacy_notice_version, created_at + ) VALUES (?, ?, ?, ?, 'active', ?, 'double_opt_in', ?, ?, ?) + `, id, record.Email, record.Email, record.Audience, record.ConsentedAt.UTC().Format(time.RFC3339), record.ConsentSource, record.PrivacyNoticeVersion, importedAt.UTC().Format(time.RFC3339)) + if err != nil { + return ImportResult{}, fmt.Errorf("insert consent record: %w", err) + } + result.Inserted++ + case err != nil: + return ImportResult{}, fmt.Errorf("find consent record: %w", err) + case existingStatus == "suppressed": + result.SuppressedSkipped++ + case existingStatus == "active": + existingTime, parseErr := time.Parse(time.RFC3339, existingConsentedAt) + if parseErr != nil { + return ImportResult{}, errors.New("stored consent timestamp is invalid") + } + if record.ConsentedAt.Before(existingTime) { + return ImportResult{}, fmt.Errorf("consent import for %s is older than the stored record", record.Audience) + } + _, err = tx.ExecContext(ctx, ` + UPDATE subscriptions + SET email = ?, consented_at = ?, consent_source = ?, privacy_notice_version = ?, updated_at = ? + WHERE id = ? + `, record.Email, record.ConsentedAt.UTC().Format(time.RFC3339), record.ConsentSource, record.PrivacyNoticeVersion, importedAt.UTC().Format(time.RFC3339), existingID) + if err != nil { + return ImportResult{}, fmt.Errorf("update consent record: %w", err) + } + result.Updated++ + default: + return ImportResult{}, fmt.Errorf("stored subscription has unsupported status %q", existingStatus) + } + } + if err := tx.Commit(); err != nil { + return ImportResult{}, fmt.Errorf("commit consent import: %w", err) + } + return result, nil +} + +// SuppressAddress records a verified provider feedback event and suppresses +// every active audience membership for that address. The separate address +// suppression also prevents a later consent import from silently reactivating +// an address that hard-bounced, complained, or was provider-suppressed. +func (registry *Registry) SuppressAddress(ctx context.Context, email, reason, eventID, eventType string, occurredAt, processedAt time.Time) (FeedbackResult, error) { + address, err := canonicalEmail(email) + if err != nil { + return FeedbackResult{}, err + } + if reason != "bounced" && reason != "complaint" && reason != "invalid" { + return FeedbackResult{}, errors.New("provider feedback reason is unsupported") + } + eventID = strings.TrimSpace(eventID) + eventType = strings.TrimSpace(eventType) + if eventID == "" || len(eventID) > 256 || strings.ContainsAny(eventID, "\r\n") || eventType == "" || len(eventType) > 120 || strings.ContainsAny(eventType, "\r\n") { + return FeedbackResult{}, errors.New("provider feedback identity is invalid") + } + if occurredAt.IsZero() || processedAt.IsZero() || occurredAt.After(processedAt.Add(5*time.Minute)) { + return FeedbackResult{}, errors.New("provider feedback timestamps are invalid") + } + tx, err := registry.db.BeginTx(ctx, nil) + if err != nil { + return FeedbackResult{}, fmt.Errorf("begin provider feedback: %w", err) + } + defer tx.Rollback() + result, err := tx.ExecContext(ctx, ` + INSERT INTO provider_events (event_id, event_type, occurred_at, processed_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(event_id) DO NOTHING + `, eventID, eventType, occurredAt.UTC().Format(time.RFC3339), processedAt.UTC().Format(time.RFC3339)) + if err != nil { + return FeedbackResult{}, fmt.Errorf("record provider feedback event: %w", err) + } + inserted, err := result.RowsAffected() + if err != nil { + return FeedbackResult{}, fmt.Errorf("inspect provider feedback insert: %w", err) + } + if inserted == 0 { + if err := tx.Commit(); err != nil { + return FeedbackResult{}, err + } + return FeedbackResult{Duplicate: true}, nil + } + _, err = tx.ExecContext(ctx, ` + INSERT INTO address_suppressions (email_normalized, reason, suppressed_at, event_id) + VALUES (?, ?, ?, ?) + ON CONFLICT(email_normalized) DO NOTHING + `, address, reason, occurredAt.UTC().Format(time.RFC3339), eventID) + if err != nil { + return FeedbackResult{}, fmt.Errorf("record address suppression: %w", err) + } + updated, err := tx.ExecContext(ctx, ` + UPDATE subscriptions + SET status = 'suppressed', suppression_reason = ?, suppressed_at = ?, updated_at = ? + WHERE email_normalized = ? AND status = 'active' + `, reason, occurredAt.UTC().Format(time.RFC3339), processedAt.UTC().Format(time.RFC3339), address) + if err != nil { + return FeedbackResult{}, fmt.Errorf("apply provider feedback suppression: %w", err) + } + stopped, err := updated.RowsAffected() + if err != nil { + return FeedbackResult{}, fmt.Errorf("count provider feedback suppressions: %w", err) + } + if err := tx.Commit(); err != nil { + return FeedbackResult{}, fmt.Errorf("commit provider feedback: %w", err) + } + return FeedbackResult{SubscriptionsStopped: stopped}, nil +} + +func (registry *Registry) ExportAudience(ctx context.Context, audience string) ([]byte, []byte, error) { + if err := validateAudience(audience); err != nil { + return nil, nil, err + } + rows, err := registry.db.QueryContext(ctx, ` + SELECT id, email, audience, status, consented_at, consent_source, privacy_notice_version, + COALESCE(suppression_reason, ''), COALESCE(suppressed_at, '') + FROM subscriptions + WHERE audience = ? + ORDER BY email_normalized + `, audience) + if err != nil { + return nil, nil, fmt.Errorf("list audience subscriptions: %w", err) + } + defer rows.Close() + var active [][]string + suppressedByAddress := map[string][]string{} + for rows.Next() { + var record storedSubscription + var suppressionReason, suppressedAt string + if err := rows.Scan(&record.ID, &record.Email, &record.Audience, &record.Status, &record.ConsentedAt, &record.ConsentSource, &record.PrivacyNoticeVersion, &suppressionReason, &suppressedAt); err != nil { + return nil, nil, err + } + switch record.Status { + case "active": + token, err := registry.tokenFor(record.ID) + if err != nil { + return nil, nil, err + } + unsubscribeURL := *registry.unsubscribeBase + query := unsubscribeURL.Query() + query.Set("token", token) + unsubscribeURL.RawQuery = query.Encode() + active = append(active, []string{record.Email, record.Audience, "active", record.ConsentedAt, "double_opt_in", record.ConsentSource, record.PrivacyNoticeVersion, unsubscribeURL.String()}) + case "suppressed": + suppressedByAddress[record.Email] = []string{record.Email, suppressionReason, suppressedAt} + default: + return nil, nil, fmt.Errorf("stored subscription has unsupported status %q", record.Status) + } + } + if err := rows.Err(); err != nil { + return nil, nil, err + } + if err := rows.Close(); err != nil { + return nil, nil, err + } + globalRows, err := registry.db.QueryContext(ctx, ` + SELECT email_normalized, reason, suppressed_at + FROM address_suppressions + ORDER BY email_normalized + `) + if err != nil { + return nil, nil, fmt.Errorf("list global address suppressions: %w", err) + } + defer globalRows.Close() + globallySuppressed := map[string]bool{} + for globalRows.Next() { + var email, reason, suppressedAt string + if err := globalRows.Scan(&email, &reason, &suppressedAt); err != nil { + return nil, nil, err + } + globallySuppressed[email] = true + suppressedByAddress[email] = []string{email, reason, suppressedAt} + } + if err := globalRows.Err(); err != nil { + return nil, nil, err + } + if len(globallySuppressed) > 0 { + filtered := active[:0] + for _, row := range active { + if !globallySuppressed[row[0]] { + filtered = append(filtered, row) + } + } + active = filtered + } + suppressedEmails := make([]string, 0, len(suppressedByAddress)) + for email := range suppressedByAddress { + suppressedEmails = append(suppressedEmails, email) + } + sort.Strings(suppressedEmails) + suppressed := make([][]string, 0, len(suppressedEmails)) + for _, email := range suppressedEmails { + suppressed = append(suppressed, suppressedByAddress[email]) + } + subscriberCSV, err := encodeCSV(deliveryHeader, active) + if err != nil { + return nil, nil, err + } + suppressionCSV, err := encodeCSV(suppressionHeader, suppressed) + if err != nil { + return nil, nil, err + } + return subscriberCSV, suppressionCSV, nil +} + +func WritePrivateExports(subscriberPath string, subscriberCSV []byte, suppressionPath string, suppressionCSV []byte) error { + subscriberTarget, err := preparePrivateTarget(subscriberPath) + if err != nil { + return fmt.Errorf("subscriber export: %w", err) + } + suppressionTarget, err := preparePrivateTarget(suppressionPath) + if err != nil { + return fmt.Errorf("suppression export: %w", err) + } + if strings.EqualFold(subscriberTarget, suppressionTarget) { + return errors.New("subscriber and suppression exports must use different paths") + } + if err := writePrivateFile(subscriberTarget, subscriberCSV); err != nil { + return fmt.Errorf("write subscriber export: %w", err) + } + if err := writePrivateFile(suppressionTarget, suppressionCSV); err != nil { + _ = os.Remove(subscriberTarget) + return fmt.Errorf("write suppression export: %w", err) + } + return nil +} + +func (registry *Registry) Unsubscribe(ctx context.Context, token string, stoppedAt time.Time) (UnsubscribeResult, error) { + if stoppedAt.IsZero() { + return UnsubscribeResult{}, errors.New("unsubscribe time is required") + } + payload, err := registry.verifyToken(token) + if err != nil { + return UnsubscribeResult{}, err + } + tx, err := registry.db.BeginTx(ctx, nil) + if err != nil { + return UnsubscribeResult{}, err + } + defer tx.Rollback() + var status string + err = tx.QueryRowContext(ctx, `SELECT status FROM subscriptions WHERE id = ?`, payload.ID).Scan(&status) + if errors.Is(err, sql.ErrNoRows) { + return UnsubscribeResult{}, ErrInvalidToken + } + if err != nil { + return UnsubscribeResult{}, fmt.Errorf("lookup unsubscribe token: %w", err) + } + if status == "suppressed" { + if err := tx.Commit(); err != nil { + return UnsubscribeResult{}, err + } + return UnsubscribeResult{AlreadyStopped: true}, nil + } + if status != "active" { + return UnsubscribeResult{}, ErrInvalidToken + } + _, err = tx.ExecContext(ctx, ` + UPDATE subscriptions + SET status = 'suppressed', suppression_reason = 'unsubscribed', suppressed_at = ?, updated_at = ? + WHERE id = ? AND status = 'active' + `, stoppedAt.UTC().Format(time.RFC3339), stoppedAt.UTC().Format(time.RFC3339), payload.ID) + if err != nil { + return UnsubscribeResult{}, fmt.Errorf("record unsubscribe: %w", err) + } + if err := tx.Commit(); err != nil { + return UnsubscribeResult{}, err + } + return UnsubscribeResult{Changed: true}, nil +} + +func (registry *Registry) ValidateToken(token string) error { + payload, err := registry.verifyToken(token) + if err != nil { + return err + } + var exists int + err = registry.db.QueryRow(`SELECT 1 FROM subscriptions WHERE id = ?`, payload.ID).Scan(&exists) + if errors.Is(err, sql.ErrNoRows) { + return ErrInvalidToken + } + return err +} + +func (registry *Registry) tokenFor(id string) (string, error) { + return registry.purposeTokenFor(id, "tradegravity-unsubscribe-v1:") +} + +func (registry *Registry) purposeTokenFor(id, domain string) (string, error) { + payloadRaw, err := json.Marshal(tokenPayload{Version: tokenVersion, ID: id}) + if err != nil { + return "", err + } + payload := base64.RawURLEncoding.EncodeToString(payloadRaw) + mac := hmac.New(sha256.New, registry.secret) + _, _ = mac.Write([]byte(domain + payload)) + signature := base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) + return payload + "." + signature, nil +} + +func (registry *Registry) verifyToken(token string) (tokenPayload, error) { + return registry.verifyPurposeToken(token, "tradegravity-unsubscribe-v1:") +} + +func (registry *Registry) verifyPurposeToken(token, domain string) (tokenPayload, error) { + parts := strings.Split(token, ".") + if len(parts) != 2 || len(token) > 1024 { + return tokenPayload{}, ErrInvalidToken + } + mac := hmac.New(sha256.New, registry.secret) + _, _ = mac.Write([]byte(domain + parts[0])) + provided, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil || base64.RawURLEncoding.EncodeToString(provided) != parts[1] || !hmac.Equal(provided, mac.Sum(nil)) { + return tokenPayload{}, ErrInvalidToken + } + payloadRaw, err := base64.RawURLEncoding.DecodeString(parts[0]) + if err != nil || base64.RawURLEncoding.EncodeToString(payloadRaw) != parts[0] { + return tokenPayload{}, ErrInvalidToken + } + var payload tokenPayload + decoder := json.NewDecoder(bytes.NewReader(payloadRaw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&payload); err != nil { + return tokenPayload{}, ErrInvalidToken + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return tokenPayload{}, ErrInvalidToken + } + if payload.Version != tokenVersion || len(payload.ID) != 32 { + return tokenPayload{}, ErrInvalidToken + } + if _, err := hex.DecodeString(payload.ID); err != nil { + return tokenPayload{}, ErrInvalidToken + } + return payload, nil +} + +func (registry *Registry) migrate() error { + statements := []string{ + `PRAGMA foreign_keys = ON;`, + `PRAGMA journal_mode = WAL;`, + `PRAGMA busy_timeout = 5000;`, + `CREATE TABLE IF NOT EXISTS subscriptions ( + id TEXT PRIMARY KEY, + email TEXT NOT NULL, + email_normalized TEXT NOT NULL, + audience TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('active', 'suppressed')), + consented_at TEXT NOT NULL, + consent_method TEXT NOT NULL CHECK (consent_method = 'double_opt_in'), + consent_source TEXT NOT NULL, + privacy_notice_version TEXT NOT NULL, + suppression_reason TEXT, + suppressed_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT, + UNIQUE(email_normalized, audience), + CHECK ((status = 'active' AND suppression_reason IS NULL AND suppressed_at IS NULL) + OR (status = 'suppressed' AND suppression_reason IS NOT NULL AND suppressed_at IS NOT NULL)) + );`, + `CREATE INDEX IF NOT EXISTS idx_subscriptions_audience_status ON subscriptions(audience, status, email_normalized);`, + `CREATE TABLE IF NOT EXISTS provider_events ( + event_id TEXT PRIMARY KEY, + event_type TEXT NOT NULL, + occurred_at TEXT NOT NULL, + processed_at TEXT NOT NULL + );`, + `CREATE TABLE IF NOT EXISTS address_suppressions ( + email_normalized TEXT PRIMARY KEY, + reason TEXT NOT NULL CHECK (reason IN ('bounced', 'complaint', 'invalid')), + suppressed_at TEXT NOT NULL, + event_id TEXT NOT NULL UNIQUE REFERENCES provider_events(event_id) + );`, + `CREATE TABLE IF NOT EXISTS pending_subscriptions ( + id TEXT PRIMARY KEY, + email TEXT NOT NULL, + email_normalized TEXT NOT NULL, + audience TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('dispatch_pending', 'sent', 'confirmed')), + consent_source TEXT NOT NULL, + privacy_notice_version TEXT NOT NULL, + requested_at TEXT NOT NULL, + expires_at TEXT NOT NULL, + last_dispatch_at TEXT, + provider_message_id TEXT, + confirmed_at TEXT, + UNIQUE(email_normalized, audience), + CHECK ((status = 'dispatch_pending' AND provider_message_id IS NULL AND confirmed_at IS NULL) + OR (status = 'sent' AND provider_message_id IS NOT NULL AND confirmed_at IS NULL) + OR (status = 'confirmed' AND confirmed_at IS NOT NULL)) + );`, + `CREATE INDEX IF NOT EXISTS idx_pending_subscriptions_expiry ON pending_subscriptions(status, expires_at);`, + } + for _, statement := range statements { + if _, err := registry.db.Exec(statement); err != nil { + return fmt.Errorf("migrate subscription database: %w", err) + } + } + return nil +} + +func parseConsentCSV(raw []byte, importedAt time.Time) ([]consentRecord, error) { + reader := csv.NewReader(bytes.NewReader(raw)) + reader.FieldsPerRecord = len(consentHeader) + rows, err := reader.ReadAll() + if err != nil { + return nil, fmt.Errorf("parse consent CSV: %w", err) + } + if len(rows) < 2 || !equalStrings(rows[0], consentHeader) { + return nil, fmt.Errorf("consent CSV header must be exactly %s and contain at least one row", strings.Join(consentHeader, ",")) + } + seen := map[string]bool{} + records := make([]consentRecord, 0, len(rows)-1) + for index, row := range rows[1:] { + line := index + 2 + for _, value := range row { + if value != strings.TrimSpace(value) { + return nil, fmt.Errorf("consent CSV line %d contains leading or trailing whitespace", line) + } + } + email, err := canonicalEmail(row[0]) + if err != nil { + return nil, fmt.Errorf("consent CSV line %d: %w", line, err) + } + if err := validateAudience(row[1]); err != nil { + return nil, fmt.Errorf("consent CSV line %d: %w", line, err) + } + key := email + "\x00" + row[1] + if seen[key] { + return nil, fmt.Errorf("consent CSV line %d duplicates an email and audience", line) + } + seen[key] = true + if row[2] != "active" || row[4] != "double_opt_in" { + return nil, fmt.Errorf("consent CSV line %d is not active double opt-in consent", line) + } + consentedAt, err := time.Parse(time.RFC3339, row[3]) + if err != nil || consentedAt.After(importedAt) { + return nil, fmt.Errorf("consent CSV line %d has invalid or future consent time", line) + } + if err := validateLabel(row[5]); err != nil { + return nil, fmt.Errorf("consent CSV line %d has invalid consent source", line) + } + if err := validateLabel(row[6]); err != nil { + return nil, fmt.Errorf("consent CSV line %d has invalid privacy notice version", line) + } + records = append(records, consentRecord{Email: email, Audience: row[1], ConsentedAt: consentedAt, ConsentSource: row[5], PrivacyNoticeVersion: row[6]}) + } + sort.Slice(records, func(first, second int) bool { + if records[first].Audience == records[second].Audience { + return records[first].Email < records[second].Email + } + return records[first].Audience < records[second].Audience + }) + return records, nil +} + +func encodeCSV(header []string, rows [][]string) ([]byte, error) { + var output bytes.Buffer + writer := csv.NewWriter(&output) + if err := writer.Write(header); err != nil { + return nil, err + } + writer.WriteAll(rows) + writer.Flush() + if err := writer.Error(); err != nil { + return nil, err + } + return output.Bytes(), nil +} + +func normalizePublicBaseURL(value string) (*url.URL, error) { + parsed, err := url.Parse(strings.TrimSpace(value)) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return nil, errors.New("public subscription base URL must be absolute HTTPS without credentials, query, or fragment") + } + if !strings.HasSuffix(parsed.Path, "/") { + parsed.Path += "/" + } + return parsed.ResolveReference(&url.URL{Path: "unsubscribe"}), nil +} + +func preparePrivateTarget(path string) (string, error) { + path = strings.TrimSpace(path) + if path == "" { + return "", errors.New("output path is required") + } + absolute, err := filepath.Abs(path) + if err != nil { + return "", err + } + parent := filepath.Dir(absolute) + if err := os.MkdirAll(parent, 0o700); err != nil { + return "", err + } + resolvedParent, err := filepath.EvalSymlinks(parent) + if err != nil { + return "", err + } + return filepath.Join(resolvedParent, filepath.Base(absolute)), nil +} + +func writePrivateFile(path string, content []byte) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + if errors.Is(err, os.ErrExist) { + return errors.New("output already exists") + } + return err + } + writeErr := func() error { + if _, err := file.Write(content); err != nil { + return err + } + return file.Sync() + }() + closeErr := file.Close() + if writeErr != nil || closeErr != nil { + _ = os.Remove(path) + if writeErr != nil { + return writeErr + } + return closeErr + } + return nil +} + +func canonicalEmail(value string) (string, error) { + if value == "" || value != strings.TrimSpace(value) || strings.Count(value, "@") != 1 { + return "", errors.New("email address is empty or malformed") + } + for _, char := range value { + if char > unicode.MaxASCII || unicode.IsSpace(char) || unicode.IsControl(char) { + return "", errors.New("email address must use a plain ASCII addr-spec") + } + } + parsed, err := mail.ParseAddress(value) + if err != nil || parsed.Address != value { + return "", errors.New("email address must not contain a display name or invalid syntax") + } + parts := strings.Split(value, "@") + if parts[0] == "" || parts[1] == "" || !strings.Contains(parts[1], ".") { + return "", errors.New("email address must contain a valid-looking domain") + } + return strings.ToLower(value), nil +} + +func validateAudience(value string) error { + if value == "" || value != strings.TrimSpace(value) || len([]rune(value)) > 120 || strings.ContainsAny(value, "@/\\") || strings.Contains(value, "://") { + return errors.New("audience must be a non-sensitive label") + } + return validateLabel(value) +} + +func validateLabel(value string) error { + if value == "" || len([]rune(value)) > 120 { + return errors.New("label is empty or too long") + } + for _, char := range value { + if unicode.IsControl(char) { + return errors.New("label contains a control character") + } + } + return nil +} + +func randomID() (string, error) { + value := make([]byte, 16) + if _, err := rand.Read(value); err != nil { + return "", fmt.Errorf("generate subscription ID: %w", err) + } + return hex.EncodeToString(value), nil +} + +func equalStrings(first, second []string) bool { + if len(first) != len(second) { + return false + } + for index := range first { + if first[index] != second[index] { + return false + } + } + return true +} diff --git a/internal/subscriptions/registry_test.go b/internal/subscriptions/registry_test.go new file mode 100644 index 0000000..50b74e8 --- /dev/null +++ b/internal/subscriptions/registry_test.go @@ -0,0 +1,251 @@ +package subscriptions + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/csv" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "tradegravity/internal/deliverypreflight" + "tradegravity/internal/distributionkit" +) + +func TestRegistryExportsOpaqueLinksAndGETDoesNotUnsubscribe(t *testing.T) { + root, registry := newTestRegistry(t) + importedAt := time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC) + result, err := registry.ImportConsents(context.Background(), testConsentCSV(), importedAt) + if err != nil { + t.Fatalf("ImportConsents() error = %v", err) + } + if result.Inserted != 2 || result.Updated != 0 || result.SuppressedSkipped != 0 { + t.Fatalf("unexpected import result: %+v", result) + } + + subscribers, suppressions, err := registry.ExportAudience(context.Background(), "pilot-audience") + if err != nil { + t.Fatal(err) + } + subscriberRows := readCSV(t, subscribers) + if len(subscriberRows) != 3 || len(readCSV(t, suppressions)) != 1 { + t.Fatalf("unexpected export rows: subscribers=%d suppressions=%d", len(subscriberRows), len(readCSV(t, suppressions))) + } + if !equalStrings(subscriberRows[0], deliveryHeader) || subscriberRows[1][0] != "alpha@example.invalid" { + t.Fatalf("unexpected subscriber export: %v", subscriberRows) + } + unsubscribeURL, err := url.Parse(subscriberRows[1][7]) + if err != nil || unsubscribeURL.Scheme != "https" || unsubscribeURL.Path != "/service/unsubscribe" { + t.Fatalf("unexpected unsubscribe URL: %q err=%v", subscriberRows[1][7], err) + } + token := unsubscribeURL.Query().Get("token") + parts := strings.Split(token, ".") + if len(parts) != 2 { + t.Fatalf("unexpected token shape: %q", token) + } + payload, err := base64.RawURLEncoding.DecodeString(parts[0]) + if err != nil || bytes.Contains(payload, []byte("alpha@example.invalid")) || bytes.Contains(payload, []byte("pilot-audience")) { + t.Fatalf("token payload leaked subscriber identity: %s", payload) + } + if err := registry.ValidateToken(token); err != nil { + t.Fatalf("ValidateToken() error = %v", err) + } + mutated := token[:len(token)-1] + "x" + if err := registry.ValidateToken(mutated); err == nil { + t.Fatal("ValidateToken() accepted a modified signature") + } + + fixedNow := time.Date(2026, 7, 17, 12, 30, 0, 0, time.UTC) + handler := registry.Handler(func() time.Time { return fixedNow }) + get := httptest.NewRequest(http.MethodGet, unsubscribeURL.RequestURI(), nil) + getResponse := httptest.NewRecorder() + handler.ServeHTTP(getResponse, get) + if getResponse.Code != http.StatusOK || !strings.Contains(getResponse.Body.String(), "has not changed") || !strings.Contains(getResponse.Body.String(), `action="/service/unsubscribe?token=`) || strings.Contains(getResponse.Body.String(), "alpha@example.invalid") { + t.Fatalf("unsafe GET response: code=%d body=%s", getResponse.Code, getResponse.Body.String()) + } + for _, header := range []string{"Cache-Control", "Content-Security-Policy", "Referrer-Policy", "X-Content-Type-Options", "X-Frame-Options"} { + if getResponse.Header().Get(header) == "" { + t.Fatalf("GET response omitted security header %s", header) + } + } + activeAfterGET, suppressedAfterGET, err := registry.ExportAudience(context.Background(), "pilot-audience") + if err != nil || len(readCSV(t, activeAfterGET)) != 3 || len(readCSV(t, suppressedAfterGET)) != 1 { + t.Fatalf("GET changed subscription state: err=%v", err) + } + + badPost := httptest.NewRequest(http.MethodPost, unsubscribeURL.RequestURI(), strings.NewReader("List-Unsubscribe=One-Click")) + badPostResponse := httptest.NewRecorder() + handler.ServeHTTP(badPostResponse, badPost) + if badPostResponse.Code != http.StatusUnsupportedMediaType { + t.Fatalf("POST without form content type = %d", badPostResponse.Code) + } + + post := httptest.NewRequest(http.MethodPost, unsubscribeURL.RequestURI(), strings.NewReader("List-Unsubscribe=One-Click")) + post.Header.Set("Content-Type", "application/x-www-form-urlencoded") + postResponse := httptest.NewRecorder() + handler.ServeHTTP(postResponse, post) + if postResponse.Code != http.StatusOK || !strings.Contains(postResponse.Body.String(), "now suppressed") || strings.Contains(postResponse.Body.String(), "alpha@example.invalid") { + t.Fatalf("unexpected POST response: code=%d body=%s", postResponse.Code, postResponse.Body.String()) + } + + repeat := httptest.NewRequest(http.MethodPost, unsubscribeURL.RequestURI(), strings.NewReader("List-Unsubscribe=One-Click")) + repeat.Header.Set("Content-Type", "application/x-www-form-urlencoded") + repeatResponse := httptest.NewRecorder() + handler.ServeHTTP(repeatResponse, repeat) + if repeatResponse.Code != http.StatusOK { + t.Fatalf("idempotent POST response = %d", repeatResponse.Code) + } + + active, suppressed, err := registry.ExportAudience(context.Background(), "pilot-audience") + if err != nil { + t.Fatal(err) + } + if len(readCSV(t, active)) != 2 || len(readCSV(t, suppressed)) != 2 { + t.Fatalf("unsubscribe export counts are wrong: active=%d suppressed=%d", len(readCSV(t, active)), len(readCSV(t, suppressed))) + } + suppressionRows := readCSV(t, suppressed) + if suppressionRows[1][0] != "alpha@example.invalid" || suppressionRows[1][1] != "unsubscribed" || suppressionRows[1][2] != fixedNow.Format(time.RFC3339) { + t.Fatalf("unexpected suppression row: %v", suppressionRows[1]) + } + + reimported, err := registry.ImportConsents(context.Background(), testConsentCSV(), importedAt.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if reimported.Updated != 1 || reimported.SuppressedSkipped != 1 || reimported.Inserted != 0 { + t.Fatalf("suppression was not durable across import: %+v", reimported) + } + + if runtime.GOOS != "windows" { + info, err := os.Stat(filepath.Join(root, "subscriptions.db")) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("database mode = %v err=%v", info.Mode().Perm(), err) + } + } +} + +func TestRegistryExportsFeedEmailPreflightWithoutLeakingTokens(t *testing.T) { + root, registry := newTestRegistry(t) + if _, err := registry.ImportConsents(context.Background(), testConsentCSV(), time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + subscribers, suppressions, err := registry.ExportAudience(context.Background(), "pilot-audience") + if err != nil { + t.Fatal(err) + } + subscriberPath := filepath.Join(root, "subscribers.csv") + suppressionPath := filepath.Join(root, "suppressions.csv") + if err := os.WriteFile(subscriberPath, subscribers, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(suppressionPath, suppressions, 0o600); err != nil { + t.Fatal(err) + } + + briefing, err := os.ReadFile(filepath.Join("..", "..", "examples", "sample-data", "briefing.json")) + if err != nil { + t.Fatal(err) + } + bundle, err := distributionkit.Build(briefing, "https://example.org/TradeGravity/") + if err != nil { + t.Fatal(err) + } + kitDir := filepath.Join(root, "kit") + if err := distributionkit.Write(kitDir, bundle); err != nil { + t.Fatal(err) + } + _, approvalRaw, err := distributionkit.Approve(kitDir, distributionkit.ApprovalRequest{ + Reviewer: "reviewer", Audience: "pilot-audience", Channels: []string{"email"}, + ApprovedAt: time.Date(2026, 7, 17, 12, 10, 0, 0, time.UTC), Attested: true, + }) + if err != nil { + t.Fatal(err) + } + if err := distributionkit.WriteApproval(kitDir, approvalRaw); err != nil { + t.Fatal(err) + } + preflight, err := deliverypreflight.Build(deliverypreflight.Request{ + KitDir: kitDir, SubscriberCSV: subscriberPath, SuppressionCSV: suppressionPath, + GeneratedAt: time.Date(2026, 7, 17, 12, 30, 0, 0, time.UTC), MaxRecipients: 25, + }) + if err != nil { + t.Fatal(err) + } + if preflight.Plan.Counts.Eligible != 2 || !preflight.Plan.UnsubscribeURLsValidated || preflight.Plan.DeliveryAuthorized { + t.Fatalf("unexpected registry-to-preflight result: %+v", preflight.Plan) + } + if bytes.Contains(preflight.JSON, []byte("example.invalid")) || bytes.Contains(preflight.JSON, []byte("token=")) { + t.Fatal("aggregate preflight leaked an address or unsubscribe token") + } +} + +func TestOpenRejectsShortSecretAndInsecureURL(t *testing.T) { + path := filepath.Join(t.TempDir(), "subscriptions.db") + if _, err := Open(path, []byte("short"), "https://subscriptions.example.invalid/"); err == nil { + t.Fatal("Open() accepted a short secret") + } + if _, err := Open(path, bytes.Repeat([]byte("s"), 32), "http://subscriptions.example.invalid/"); err == nil { + t.Fatal("Open() accepted an insecure public URL") + } +} + +func TestWritePrivateExportsRefusesOverwrite(t *testing.T) { + root := t.TempDir() + subscriberPath := filepath.Join(root, "private", "subscribers.csv") + suppressionPath := filepath.Join(root, "private", "suppressions.csv") + if err := WritePrivateExports(subscriberPath, []byte("subscribers\n"), suppressionPath, []byte("suppressions\n")); err != nil { + t.Fatalf("WritePrivateExports() error = %v", err) + } + if err := WritePrivateExports(subscriberPath, []byte("changed\n"), suppressionPath, []byte("changed\n")); err == nil { + t.Fatal("WritePrivateExports() overwrote existing private files") + } + stored, err := os.ReadFile(subscriberPath) + if err != nil || string(stored) != "subscribers\n" { + t.Fatalf("subscriber export changed: %q err=%v", stored, err) + } + if runtime.GOOS != "windows" { + info, err := os.Stat(suppressionPath) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("private export mode = %v", info.Mode().Perm()) + } + } +} + +func newTestRegistry(t *testing.T) (string, *Registry) { + t.Helper() + root := t.TempDir() + registry, err := Open(filepath.Join(root, "subscriptions.db"), bytes.Repeat([]byte("s"), 32), "https://subscriptions.example.invalid/service/") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = registry.Close() }) + return root, registry +} + +func testConsentCSV() []byte { + return []byte(strings.Join([]string{ + "email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version", + "alpha@example.invalid,pilot-audience,active,2026-07-10T01:00:00Z,double_opt_in,test-form,v1", + "beta@example.invalid,pilot-audience,active,2026-07-11T01:00:00Z,double_opt_in,test-form,v1", + "", + }, "\n")) +} + +func readCSV(t *testing.T, raw []byte) [][]string { + t.Helper() + rows, err := csv.NewReader(bytes.NewReader(raw)).ReadAll() + if err != nil { + t.Fatal(err) + } + return rows +} diff --git a/internal/subscriptions/signup.go b/internal/subscriptions/signup.go new file mode 100644 index 0000000..04c455a --- /dev/null +++ b/internal/subscriptions/signup.go @@ -0,0 +1,283 @@ +package subscriptions + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/url" + "strings" + "time" +) + +const confirmationTokenDomain = "tradegravity-confirmation-v1:" + +var ErrInvalidConfirmation = errors.New("invalid or expired confirmation token") + +type SignupConfig struct { + Audience string + ConsentSource string + PrivacyNoticeVersion string + PrivacyNoticeURL string + ConfirmationTTL time.Duration + DispatchCooldown time.Duration + MaxPending int +} + +type ConfirmationDispatch struct { + PendingID string + Email string + ConfirmationURL string + IdempotencyKey string + ExpiresAt time.Time + ShouldDispatch bool +} + +type ConfirmResult struct { + Activated bool + AlreadyActive bool +} + +func validateSignupConfig(config SignupConfig) error { + if err := validateAudience(config.Audience); err != nil { + return err + } + if err := validateLabel(config.ConsentSource); err != nil { + return fmt.Errorf("consent source: %w", err) + } + if err := validateLabel(config.PrivacyNoticeVersion); err != nil { + return fmt.Errorf("privacy notice version: %w", err) + } + privacy, err := url.Parse(strings.TrimSpace(config.PrivacyNoticeURL)) + if err != nil || privacy.Scheme != "https" || privacy.Host == "" || privacy.User != nil { + return errors.New("privacy notice URL must be absolute HTTPS") + } + if config.ConfirmationTTL < 5*time.Minute || config.ConfirmationTTL > 24*time.Hour { + return errors.New("confirmation TTL must be between 5 minutes and 24 hours") + } + if config.DispatchCooldown < 30*time.Second || config.DispatchCooldown > config.ConfirmationTTL { + return errors.New("dispatch cooldown is invalid") + } + if config.MaxPending < 1 || config.MaxPending > 10000 { + return errors.New("maximum pending subscriptions is invalid") + } + return nil +} + +func (registry *Registry) RequestSubscription(ctx context.Context, email string, config SignupConfig, requestedAt time.Time) (ConfirmationDispatch, error) { + if err := validateSignupConfig(config); err != nil { + return ConfirmationDispatch{}, err + } + address, err := canonicalEmail(email) + if err != nil || requestedAt.IsZero() { + return ConfirmationDispatch{}, errors.New("subscription request is invalid") + } + now := requestedAt.UTC() + tx, err := registry.db.BeginTx(ctx, nil) + if err != nil { + return ConfirmationDispatch{}, err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `DELETE FROM pending_subscriptions WHERE expires_at <= ?`, now.Format(time.RFC3339)); err != nil { + return ConfirmationDispatch{}, fmt.Errorf("expire pending subscriptions: %w", err) + } + var marker int + if err = tx.QueryRowContext(ctx, `SELECT 1 FROM address_suppressions WHERE email_normalized = ?`, address).Scan(&marker); err == nil { + _ = tx.Commit() + return ConfirmationDispatch{}, nil + } else if !errors.Is(err, sql.ErrNoRows) { + return ConfirmationDispatch{}, err + } + var subscriptionStatus string + err = tx.QueryRowContext(ctx, `SELECT status FROM subscriptions WHERE email_normalized = ? AND audience = ?`, address, config.Audience).Scan(&subscriptionStatus) + if err == nil && subscriptionStatus == "active" { + _ = tx.Commit() + return ConfirmationDispatch{}, nil + } + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return ConfirmationDispatch{}, err + } + if err == nil && subscriptionStatus != "suppressed" { + return ConfirmationDispatch{}, errors.New("stored subscription status is invalid") + } + if subscriptionStatus == "suppressed" { + var reason string + if err := tx.QueryRowContext(ctx, `SELECT suppression_reason FROM subscriptions WHERE email_normalized = ? AND audience = ?`, address, config.Audience).Scan(&reason); err != nil { + return ConfirmationDispatch{}, err + } + if reason != "unsubscribed" { + _ = tx.Commit() + return ConfirmationDispatch{}, nil + } + } + var id, status, expiresRaw string + var lastDispatch sql.NullString + err = tx.QueryRowContext(ctx, `SELECT id, status, expires_at, last_dispatch_at FROM pending_subscriptions WHERE email_normalized = ? AND audience = ?`, address, config.Audience).Scan(&id, &status, &expiresRaw, &lastDispatch) + if err == nil { + expires, parseErr := time.Parse(time.RFC3339, expiresRaw) + if parseErr != nil { + return ConfirmationDispatch{}, errors.New("stored confirmation expiry is invalid") + } + if status == "confirmed" || expires.Before(now) || expires.Equal(now) { + if _, err := tx.ExecContext(ctx, `DELETE FROM pending_subscriptions WHERE id = ?`, id); err != nil { + return ConfirmationDispatch{}, err + } + } else if status == "sent" { + _ = tx.Commit() + return ConfirmationDispatch{}, nil + } else if status == "dispatch_pending" { + if lastDispatch.Valid { + last, parseErr := time.Parse(time.RFC3339, lastDispatch.String) + if parseErr != nil { + return ConfirmationDispatch{}, errors.New("stored dispatch time is invalid") + } + if now.Before(last.Add(config.DispatchCooldown)) { + _ = tx.Commit() + return ConfirmationDispatch{}, nil + } + } + return registry.prepareDispatch(ctx, tx, id, address, expires, now) + } else { + return ConfirmationDispatch{}, errors.New("stored pending status is invalid") + } + } else if !errors.Is(err, sql.ErrNoRows) { + return ConfirmationDispatch{}, err + } + var pendingCount int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM pending_subscriptions WHERE status != 'confirmed' AND expires_at > ?`, now.Format(time.RFC3339)).Scan(&pendingCount); err != nil { + return ConfirmationDispatch{}, err + } + if pendingCount >= config.MaxPending { + _ = tx.Commit() + return ConfirmationDispatch{}, nil + } + id, err = randomID() + if err != nil { + return ConfirmationDispatch{}, err + } + expires := now.Add(config.ConfirmationTTL) + _, err = tx.ExecContext(ctx, `INSERT INTO pending_subscriptions (id,email,email_normalized,audience,status,consent_source,privacy_notice_version,requested_at,expires_at) VALUES (?,?,?,?,'dispatch_pending',?,?,?,?)`, id, address, address, config.Audience, config.ConsentSource, config.PrivacyNoticeVersion, now.Format(time.RFC3339), expires.Format(time.RFC3339)) + if err != nil { + return ConfirmationDispatch{}, fmt.Errorf("create pending subscription: %w", err) + } + return registry.prepareDispatch(ctx, tx, id, address, expires, now) +} + +func (registry *Registry) prepareDispatch(ctx context.Context, tx *sql.Tx, id, email string, expires, now time.Time) (ConfirmationDispatch, error) { + if _, err := tx.ExecContext(ctx, `UPDATE pending_subscriptions SET last_dispatch_at = ? WHERE id = ? AND status = 'dispatch_pending'`, now.Format(time.RFC3339), id); err != nil { + return ConfirmationDispatch{}, err + } + if err := tx.Commit(); err != nil { + return ConfirmationDispatch{}, err + } + token, err := registry.confirmationTokenFor(id) + if err != nil { + return ConfirmationDispatch{}, err + } + confirmation := *registry.unsubscribeBase + confirmation.Path = strings.TrimSuffix(confirmation.Path, "unsubscribe") + "confirm" + query := confirmation.Query() + query.Set("token", token) + confirmation.RawQuery = query.Encode() + return ConfirmationDispatch{PendingID: id, Email: email, ConfirmationURL: confirmation.String(), IdempotencyKey: "tradegravity-confirm/" + id, ExpiresAt: expires, ShouldDispatch: true}, nil +} + +func (registry *Registry) MarkConfirmationDispatched(ctx context.Context, id, providerMessageID string, dispatchedAt time.Time) error { + providerMessageID = strings.TrimSpace(providerMessageID) + if len(id) != 32 || providerMessageID == "" || len(providerMessageID) > 256 || strings.ContainsAny(providerMessageID, "\r\n@:/") || dispatchedAt.IsZero() { + return errors.New("confirmation dispatch evidence is invalid") + } + result, err := registry.db.ExecContext(ctx, `UPDATE pending_subscriptions SET status='sent', provider_message_id=? WHERE id=? AND status='dispatch_pending'`, providerMessageID, id) + if err != nil { + return err + } + changed, _ := result.RowsAffected() + if changed == 1 { + return nil + } + var status, stored string + if err := registry.db.QueryRowContext(ctx, `SELECT status, COALESCE(provider_message_id,'') FROM pending_subscriptions WHERE id=?`, id).Scan(&status, &stored); err != nil || status != "sent" || stored != providerMessageID { + return errors.New("confirmation dispatch could not be recorded") + } + return nil +} + +func (registry *Registry) ValidateConfirmation(token string, at time.Time) error { + payload, err := registry.verifyPurposeToken(token, confirmationTokenDomain) + if err != nil || at.IsZero() { + return ErrInvalidConfirmation + } + var status, expiresRaw string + if err := registry.db.QueryRow(`SELECT status, expires_at FROM pending_subscriptions WHERE id=?`, payload.ID).Scan(&status, &expiresRaw); err != nil { + return ErrInvalidConfirmation + } + expires, err := time.Parse(time.RFC3339, expiresRaw) + if err != nil || status == "confirmed" || !at.UTC().Before(expires) { + return ErrInvalidConfirmation + } + return nil +} + +func (registry *Registry) ConfirmSubscription(ctx context.Context, token string, at time.Time) (ConfirmResult, error) { + payload, err := registry.verifyPurposeToken(token, confirmationTokenDomain) + if err != nil || at.IsZero() { + return ConfirmResult{}, ErrInvalidConfirmation + } + now := at.UTC() + tx, err := registry.db.BeginTx(ctx, nil) + if err != nil { + return ConfirmResult{}, err + } + defer tx.Rollback() + var email, audience, status, source, version, expiresRaw string + err = tx.QueryRowContext(ctx, `SELECT email_normalized,audience,status,consent_source,privacy_notice_version,expires_at FROM pending_subscriptions WHERE id=?`, payload.ID).Scan(&email, &audience, &status, &source, &version, &expiresRaw) + if err != nil { + return ConfirmResult{}, ErrInvalidConfirmation + } + expires, err := time.Parse(time.RFC3339, expiresRaw) + if err != nil || !now.Before(expires) { + return ConfirmResult{}, ErrInvalidConfirmation + } + if status == "confirmed" { + _ = tx.Commit() + return ConfirmResult{AlreadyActive: true}, nil + } + var blocked int + if err := tx.QueryRowContext(ctx, `SELECT 1 FROM address_suppressions WHERE email_normalized=?`, email).Scan(&blocked); err == nil { + return ConfirmResult{}, ErrInvalidConfirmation + } else if !errors.Is(err, sql.ErrNoRows) { + return ConfirmResult{}, err + } + var subscriptionID, subscriptionStatus, reason string + err = tx.QueryRowContext(ctx, `SELECT id,status,COALESCE(suppression_reason,'') FROM subscriptions WHERE email_normalized=? AND audience=?`, email, audience).Scan(&subscriptionID, &subscriptionStatus, &reason) + switch { + case errors.Is(err, sql.ErrNoRows): + subscriptionID, err = randomID() + if err != nil { + return ConfirmResult{}, err + } + _, err = tx.ExecContext(ctx, `INSERT INTO subscriptions (id,email,email_normalized,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,created_at) VALUES (?,?,?,?,'active',?,'double_opt_in',?,?,?)`, subscriptionID, email, email, audience, now.Format(time.RFC3339), source, version, now.Format(time.RFC3339)) + case err != nil: + return ConfirmResult{}, err + case subscriptionStatus == "active": + case subscriptionStatus == "suppressed" && reason == "unsubscribed": + _, err = tx.ExecContext(ctx, `UPDATE subscriptions SET status='active',consented_at=?,consent_source=?,privacy_notice_version=?,suppression_reason=NULL,suppressed_at=NULL,updated_at=? WHERE id=?`, now.Format(time.RFC3339), source, version, now.Format(time.RFC3339), subscriptionID) + default: + return ConfirmResult{}, ErrInvalidConfirmation + } + if err != nil { + return ConfirmResult{}, err + } + _, err = tx.ExecContext(ctx, `UPDATE pending_subscriptions SET status='confirmed',confirmed_at=? WHERE id=?`, now.Format(time.RFC3339), payload.ID) + if err != nil { + return ConfirmResult{}, err + } + if err := tx.Commit(); err != nil { + return ConfirmResult{}, err + } + return ConfirmResult{Activated: true}, nil +} + +func (registry *Registry) confirmationTokenFor(id string) (string, error) { + return registry.purposeTokenFor(id, confirmationTokenDomain) +} diff --git a/internal/subscriptions/signup_test.go b/internal/subscriptions/signup_test.go new file mode 100644 index 0000000..6f29135 --- /dev/null +++ b/internal/subscriptions/signup_test.go @@ -0,0 +1,189 @@ +package subscriptions + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +type recordingConfirmationSender struct { + messages []ConfirmationEmail + failures int +} + +func (sender *recordingConfirmationSender) SendConfirmation(_ context.Context, message ConfirmationEmail) (string, error) { + sender.messages = append(sender.messages, message) + if sender.failures > 0 { + sender.failures-- + return "", errors.New("simulated provider uncertainty") + } + return "msg_test_123", nil +} + +func testSignupConfig() SignupConfig { + return SignupConfig{ + Audience: "tradegravity-briefing", ConsentSource: "public-form", PrivacyNoticeVersion: "v1", + PrivacyNoticeURL: "https://subscriptions.example.invalid/privacy", ConfirmationTTL: 30 * time.Minute, + DispatchCooldown: time.Minute, MaxPending: 100, + } +} + +func postForm(handler http.Handler, path string, values url.Values) *httptest.ResponseRecorder { + request := httptest.NewRequest(http.MethodPost, path, strings.NewReader(values.Encode())) + request.Header.Set("Content-Type", "application/x-www-form-urlencoded") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + return response +} + +func TestDoubleOptInRequiresExplicitConfirmationPOST(t *testing.T) { + _, registry := newTestRegistry(t) + now := time.Date(2026, 7, 17, 10, 0, 0, 0, time.UTC) + sender := &recordingConfirmationSender{} + handler, err := registry.HandlerWithOptions(HandlerOptions{Now: func() time.Time { return now }, Signup: &SignupOptions{Config: testSignupConfig(), Sender: sender}}) + if err != nil { + t.Fatal(err) + } + response := postForm(handler, "/service/subscribe", url.Values{"email": {"reader@example.invalid"}, "privacy": {"accepted"}}) + if response.Code != http.StatusAccepted { + t.Fatalf("signup status=%d body=%s", response.Code, response.Body.String()) + } + if len(sender.messages) != 1 { + t.Fatalf("confirmation sends=%d", len(sender.messages)) + } + if strings.Contains(response.Body.String(), "reader@example.invalid") || strings.Contains(response.Body.String(), "token=") { + t.Fatal("public response leaked subscriber identity or token") + } + confirmationURL, err := url.Parse(sender.messages[0].ConfirmationURL) + if err != nil { + t.Fatal(err) + } + if confirmationURL.Path != "/service/confirm" { + t.Fatalf("confirmation path=%s", confirmationURL.Path) + } + parts := strings.Split(confirmationURL.Query().Get("token"), ".") + if len(parts) != 2 { + t.Fatal("invalid confirmation token shape") + } + payload, err := url.QueryUnescape(parts[0]) + if err != nil { + t.Fatal(err) + } + if strings.Contains(payload, "reader") || strings.Contains(payload, "tradegravity") { + t.Fatal("confirmation token leaked identity") + } + + get := httptest.NewRequest(http.MethodGet, confirmationURL.RequestURI(), nil) + getResponse := httptest.NewRecorder() + handler.ServeHTTP(getResponse, get) + if getResponse.Code != http.StatusOK || !strings.Contains(getResponse.Body.String(), "has not activated") { + t.Fatalf("confirmation GET=%d %s", getResponse.Code, getResponse.Body.String()) + } + active, _, err := registry.ExportAudience(context.Background(), testSignupConfig().Audience) + if err != nil { + t.Fatal(err) + } + if len(readCSV(t, active)) != 1 { + t.Fatal("confirmation GET activated the subscription") + } + + confirm := postForm(handler, confirmationURL.RequestURI(), url.Values{"confirm": {"yes"}}) + if confirm.Code != http.StatusOK || strings.Contains(confirm.Body.String(), "reader@example.invalid") { + t.Fatalf("confirmation POST=%d %s", confirm.Code, confirm.Body.String()) + } + active, _, err = registry.ExportAudience(context.Background(), testSignupConfig().Audience) + if err != nil { + t.Fatal(err) + } + rows := readCSV(t, active) + if len(rows) != 2 || rows[1][0] != "reader@example.invalid" || rows[1][4] != "double_opt_in" { + t.Fatalf("active export=%v", rows) + } + repeat := postForm(handler, confirmationURL.RequestURI(), url.Values{"confirm": {"yes"}}) + if repeat.Code != http.StatusOK { + t.Fatalf("repeat confirmation=%d", repeat.Code) + } + secondSignup := postForm(handler, "/service/subscribe", url.Values{"email": {"reader@example.invalid"}, "privacy": {"accepted"}}) + if secondSignup.Code != http.StatusAccepted || len(sender.messages) != 1 { + t.Fatal("active address triggered another confirmation") + } +} + +func TestConfirmationRetryUsesStableIdempotencyKey(t *testing.T) { + _, registry := newTestRegistry(t) + now := time.Date(2026, 7, 17, 10, 0, 0, 0, time.UTC) + sender := &recordingConfirmationSender{failures: 1} + handler, err := registry.HandlerWithOptions(HandlerOptions{Now: func() time.Time { return now }, Signup: &SignupOptions{Config: testSignupConfig(), Sender: sender}}) + if err != nil { + t.Fatal(err) + } + values := url.Values{"email": {"retry@example.invalid"}, "privacy": {"accepted"}} + if got := postForm(handler, "/service/subscribe", values).Code; got != http.StatusServiceUnavailable { + t.Fatalf("first status=%d", got) + } + if got := postForm(handler, "/service/subscribe", values).Code; got != http.StatusAccepted { + t.Fatalf("cooldown status=%d", got) + } + if len(sender.messages) != 1 { + t.Fatalf("cooldown triggered send: %d", len(sender.messages)) + } + now = now.Add(61 * time.Second) + if got := postForm(handler, "/service/subscribe", values).Code; got != http.StatusAccepted { + t.Fatalf("retry status=%d", got) + } + if len(sender.messages) != 2 || sender.messages[0].IdempotencyKey != sender.messages[1].IdempotencyKey || sender.messages[0].ConfirmationURL != sender.messages[1].ConfirmationURL { + t.Fatalf("retry was not stable: %+v", sender.messages) + } +} + +func TestGlobalProviderSuppressionBlocksConfirmation(t *testing.T) { + _, registry := newTestRegistry(t) + now := time.Date(2026, 7, 17, 10, 0, 0, 0, time.UTC) + _, err := registry.SuppressAddress(context.Background(), "blocked@example.invalid", "bounced", "evt-1", "email.bounced", now.Add(-time.Minute), now) + if err != nil { + t.Fatal(err) + } + sender := &recordingConfirmationSender{} + handler, err := registry.HandlerWithOptions(HandlerOptions{Now: func() time.Time { return now }, Signup: &SignupOptions{Config: testSignupConfig(), Sender: sender}}) + if err != nil { + t.Fatal(err) + } + response := postForm(handler, "/service/subscribe", url.Values{"email": {"blocked@example.invalid"}, "privacy": {"accepted"}}) + if response.Code != http.StatusAccepted || len(sender.messages) != 0 { + t.Fatalf("suppressed signup status=%d sends=%d", response.Code, len(sender.messages)) + } +} + +func TestConfirmationExpiresAndPurposeIsSeparated(t *testing.T) { + _, registry := newTestRegistry(t) + now := time.Date(2026, 7, 17, 10, 0, 0, 0, time.UTC) + dispatch, err := registry.RequestSubscription(context.Background(), "expiry@example.invalid", testSignupConfig(), now) + if err != nil { + t.Fatal(err) + } + token := strings.TrimPrefix(dispatch.ConfirmationURL, strings.Split(dispatch.ConfirmationURL, "?")[0]+"?token=") + decoded, _ := url.QueryUnescape(token) + if registry.ValidateToken(decoded) == nil { + t.Fatal("confirmation token was accepted as unsubscribe token") + } + forged := decoded[:len(decoded)-1] + "x" + if err := registry.ValidateConfirmation(forged, now); !errors.Is(err, ErrInvalidConfirmation) { + t.Fatalf("forged confirmation validation=%v", err) + } + if err := registry.ValidateConfirmation(decoded, now.Add(31*time.Minute)); !errors.Is(err, ErrInvalidConfirmation) { + t.Fatalf("expired validation=%v", err) + } + if _, err := registry.ConfirmSubscription(context.Background(), decoded, now.Add(31*time.Minute)); !errors.Is(err, ErrInvalidConfirmation) { + t.Fatalf("expired confirmation=%v", err) + } + payload, err := base64.RawURLEncoding.DecodeString(strings.Split(decoded, ".")[0]) + if err != nil || strings.Contains(string(payload), "expiry@example.invalid") || strings.Contains(string(payload), testSignupConfig().Audience) { + t.Fatalf("confirmation token payload leaked identity: %q err=%v", payload, err) + } +} diff --git a/internal/subscriptions/webhook_test.go b/internal/subscriptions/webhook_test.go new file mode 100644 index 0000000..a0cf818 --- /dev/null +++ b/internal/subscriptions/webhook_test.go @@ -0,0 +1,144 @@ +package subscriptions + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + svix "github.com/svix/svix-webhooks/go" +) + +const testWebhookSecret = "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw" + +func TestSignedResendFeedbackSuppressesOnceAndBlocksLaterImport(t *testing.T) { + _, registry := newTestRegistry(t) + now := time.Now().UTC().Truncate(time.Second) + if _, err := registry.ImportConsents(context.Background(), testConsentCSV(), now); err != nil { + t.Fatal(err) + } + handler, err := registry.HandlerWithResendWebhook(testWebhookSecret, func() time.Time { return now }) + if err != nil { + t.Fatal(err) + } + + bounced := webhookPayload(t, "email.bounced", now, "alpha@example.invalid") + response := sendSignedWebhook(t, handler, "msg_bounced_alpha", now, bounced) + if response.Code != http.StatusNoContent || response.Body.Len() != 0 { + t.Fatalf("signed bounce response = %d %q", response.Code, response.Body.String()) + } + active, suppressed, err := registry.ExportAudience(context.Background(), "pilot-audience") + if err != nil { + t.Fatal(err) + } + if len(readCSV(t, active)) != 2 || len(readCSV(t, suppressed)) != 2 { + t.Fatalf("bounce was not applied: active=%d suppressed=%d", len(readCSV(t, active)), len(readCSV(t, suppressed))) + } + if row := readCSV(t, suppressed)[1]; row[0] != "alpha@example.invalid" || row[1] != "bounced" || row[2] != now.Format(time.RFC3339) { + t.Fatalf("unexpected bounce suppression: %v", row) + } + + repeat := sendSignedWebhook(t, handler, "msg_bounced_alpha", now, bounced) + if repeat.Code != http.StatusNoContent { + t.Fatalf("duplicate webhook response = %d", repeat.Code) + } + _, repeatedSuppressions, err := registry.ExportAudience(context.Background(), "pilot-audience") + if err != nil || len(readCSV(t, repeatedSuppressions)) != 2 { + t.Fatalf("duplicate webhook changed suppression rows: err=%v rows=%d", err, len(readCSV(t, repeatedSuppressions))) + } + + invalid := httptest.NewRequest(http.MethodPost, "/service/webhooks/resend", nil) + invalid.Header.Set("Content-Type", "application/json") + invalid.Header.Set("svix-id", "msg_invalid") + invalid.Header.Set("svix-timestamp", strconv.FormatInt(now.Unix(), 10)) + invalid.Header.Set("svix-signature", "v1,invalid") + invalidResponse := httptest.NewRecorder() + handler.ServeHTTP(invalidResponse, invalid) + if invalidResponse.Code != http.StatusBadRequest { + t.Fatalf("invalid signature response = %d", invalidResponse.Code) + } + + preexisting := webhookPayload(t, "email.suppressed", now, "gamma@example.invalid") + preexistingResponse := sendSignedWebhook(t, handler, "msg_suppressed_gamma", now, preexisting) + if preexistingResponse.Code != http.StatusNoContent { + t.Fatalf("preexisting suppression response = %d", preexistingResponse.Code) + } + gammaConsent := []byte("email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version\n" + + "gamma@example.invalid,pilot-audience,active," + now.Add(-time.Hour).Format(time.RFC3339) + ",double_opt_in,test-form,v1\n") + result, err := registry.ImportConsents(context.Background(), gammaConsent, now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + if result.Inserted != 0 || result.Updated != 0 || result.SuppressedSkipped != 1 { + t.Fatalf("provider-suppressed address was imported: %+v", result) + } + _, allSuppressions, err := registry.ExportAudience(context.Background(), "pilot-audience") + if err != nil { + t.Fatal(err) + } + rows := readCSV(t, allSuppressions) + if len(rows) != 3 || rows[2][0] != "gamma@example.invalid" || rows[2][1] != "invalid" { + t.Fatalf("global suppression was not included in the private export: %v", rows) + } +} + +func TestResendWebhookIgnoresSignedNonSuppressionEvents(t *testing.T) { + _, registry := newTestRegistry(t) + now := time.Now().UTC().Truncate(time.Second) + if _, err := registry.ImportConsents(context.Background(), testConsentCSV(), now); err != nil { + t.Fatal(err) + } + handler, err := registry.HandlerWithResendWebhook(testWebhookSecret, func() time.Time { return now }) + if err != nil { + t.Fatal(err) + } + payload := webhookPayload(t, "email.delivered", now, "alpha@example.invalid") + response := sendSignedWebhook(t, handler, "msg_delivered_alpha", now, payload) + if response.Code != http.StatusNoContent { + t.Fatalf("non-suppression event response = %d", response.Code) + } + active, suppressed, err := registry.ExportAudience(context.Background(), "pilot-audience") + if err != nil || len(readCSV(t, active)) != 3 || len(readCSV(t, suppressed)) != 1 { + t.Fatalf("non-suppression event changed state: err=%v", err) + } +} + +func webhookPayload(t *testing.T, eventType string, occurredAt time.Time, email string) []byte { + t.Helper() + payload, err := json.Marshal(map[string]any{ + "type": eventType, + "created_at": occurredAt.Format(time.RFC3339Nano), + "data": map[string]any{ + "email_id": "provider-email-id", + "to": []string{email}, + }, + }) + if err != nil { + t.Fatal(err) + } + return payload +} + +func sendSignedWebhook(t *testing.T, handler http.Handler, eventID string, timestamp time.Time, payload []byte) *httptest.ResponseRecorder { + t.Helper() + webhook, err := svix.NewWebhook(testWebhookSecret) + if err != nil { + t.Fatal(err) + } + signature, err := webhook.Sign(eventID, timestamp, payload) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "/service/webhooks/resend", bytes.NewReader(payload)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("svix-id", eventID) + request.Header.Set("svix-timestamp", strconv.FormatInt(timestamp.Unix(), 10)) + request.Header.Set("svix-signature", signature) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + return response +} diff --git a/site/app.js b/site/app.js index 94644d4..9fb91a3 100644 --- a/site/app.js +++ b/site/app.js @@ -18,6 +18,7 @@ const CATALOG_URL = "./data/catalog.json"; const SEMICONDUCTOR_REFERENCE_URL = "./data/semiconductors/reference.json"; const SEMICONDUCTOR_MONTHLY_INDEX_URL = "./data/semiconductors/monthly/index.json"; const PUBLICATION_CHANGES_URL = "./data/changes.json"; +const BRIEFING_URL = "./data/briefing.json"; const security = globalThis.TradeGravitySecurity; if (!security) { throw new Error("TradeGravity security helpers failed to load."); @@ -46,6 +47,10 @@ const newsTools = globalThis.TradeGravityNewsTools; if (!newsTools) { throw new Error("TradeGravity news helpers failed to load."); } +const briefingTools = globalThis.TradeGravityBriefingTools; +if (!briefingTools) { + throw new Error("TradeGravity briefing helpers failed to load."); +} const { encodeCSV, escapeHTML, normalizeISO2, normalizeISO3, safeHTTPSURL } = security; const { buildCSVMatrix: createCSVMatrix } = dataTools; const { @@ -87,6 +92,12 @@ const { buildGdeltURL, curateNewsArticles, } = newsTools; +const { + normalizeBriefing, + materializeEmailMarkdown, + buildCarouselBundle, + briefingFilename, +} = briefingTools; const els = { svgUSA: document.getElementById("svg-usa"), @@ -135,6 +146,12 @@ const els = { chipCountryFilter: document.getElementById("chipCountryFilter"), chipDownloadCSV: document.getElementById("chipDownloadCSV"), chipPublicationChanges: document.getElementById("chipPublicationChanges"), + chipBriefingStatus: document.getElementById("chipBriefingStatus"), + chipBriefingSignals: document.getElementById("chipBriefingSignals"), + briefingDownloadEmail: document.getElementById("briefingDownloadEmail"), + briefingDownloadCarousel: document.getElementById("briefingDownloadCarousel"), + briefingCopyLink: document.getElementById("briefingCopyLink"), + briefingDeliveryNote: document.getElementById("briefingDeliveryNote"), chipTrends: document.getElementById("chipTrends"), chipValueChain: document.getElementById("chipValueChain"), chipRoleLandscape: document.getElementById("chipRoleLandscape"), @@ -204,6 +221,7 @@ let state = { semiconductorReference: null, semiconductorMonthlyIndex: null, publicationChanges: null, + briefing: null, semiconductorMonthlyFileCache: {}, semiconductorMonthlyFile: null, semiconductorMonthlyLoading: false, @@ -2511,6 +2529,68 @@ function renderChipPublicationChanges(){ els.chipPublicationChanges.innerHTML = `
${escapeHTML(pulse.status)}${pulse.status === "changed" ? "The published observation set changed" : "No observed publication delta"}Compared with ${escapeHTML(pulse.previousGeneratedAt || "the previous release")}
New / removed months${escapeHTML(periods)}
Reporter coverage${escapeHTML(coverage)}
Added / removed rows${summary.addedRows} added · ${summary.removedRows} removed
Revised matching rows${summary.revisedRows}
Source observation delta${summary.observationDelta > 0 ? "+" : ""}${summary.observationDelta}
${revisionDetail}
This compares generated publications at identical reporter–month–classification–HS6 keys. It does not describe economic month-to-month growth, causality, or a physical shipment route.
`; } +function renderDistributionBriefing(){ + if (!els.chipBriefingSignals || !els.chipBriefingStatus) return; + const briefing = state.briefing; + const ready = briefing?.status === "ready"; + els.briefingDownloadEmail.disabled = !ready; + els.briefingDownloadCarousel.disabled = !ready; + els.briefingCopyLink.disabled = !ready; + if (!briefing) { + els.chipBriefingStatus.textContent = "Unavailable"; + els.chipBriefingStatus.className = "scopeBadge warningBadge"; + els.chipBriefingSignals.innerHTML = `
Distribution briefing unavailableThe data remains usable, but briefing.json was missing or failed its browser contract check. No email or social draft is inferred.
`; + if (els.briefingDeliveryNote) els.briefingDeliveryNote.textContent = "Delivery is not configured, and no subscriber data is collected by this static site."; + return; + } + if (!ready) { + els.chipBriefingStatus.textContent = "Insufficient monthly comparison"; + els.chipBriefingStatus.className = "scopeBadge warningBadge"; + els.chipBriefingSignals.innerHTML = `
No distribution draft generated${escapeHTML(briefing.email?.preview || "Two comparable monthly observations were not available. This is not interpreted as no change.")}
`; + return; + } + els.chipBriefingStatus.textContent = `${briefing.latest_period} · review required`; + els.chipBriefingStatus.className = "scopeBadge"; + const kindLabels = { + reporter_total_change: "Scale observation", + anchor_share_shift: "Anchor-balance observation", + product_total_change: "Product observation", + }; + els.chipBriefingSignals.innerHTML = briefing.signals.map(signal => { + const product = signal.code ? ` · HS6 ${escapeHTML(signal.code)}` : ""; + const evidenceCount = Array.isArray(signal.evidence) ? signal.evidence.length : 0; + return `
${escapeHTML(kindLabels[signal.kind] || signal.kind)}

${escapeHTML(signal.title)}

${escapeHTML(signal.summary)}

${escapeHTML(signal.previous_period)} → ${escapeHTML(signal.period)}${product} · ${evidenceCount} cited artifacts
`; + }).join(""); + if (els.briefingDeliveryNote) { + els.briefingDeliveryNote.textContent = `Edition ${briefing.edition_id}. Draft exports require human review. Delivery is not configured; this static site stores no subscriber data and does not publish to social platforms.`; + } +} + +function downloadBriefingEmail(){ + if (state.briefing?.status !== "ready") return; + const baseURL = new URL(".", window.location.href).href; + const markdown = materializeEmailMarkdown(state.briefing, baseURL); + if (!markdown) return; + downloadBlob(new Blob([markdown + "\n"], { type: "text/markdown;charset=utf-8" }), briefingFilename(state.briefing, "email-draft", "md")); +} + +function downloadBriefingCarousel(){ + if (state.briefing?.status !== "ready") return; + const evidenceBaseURL = new URL("./data/", window.location.href).href; + const bundle = buildCarouselBundle(state.briefing, evidenceBaseURL); + if (!bundle) return; + downloadBlob(new Blob([JSON.stringify(bundle, null, 2) + "\n"], { type: "application/json;charset=utf-8" }), briefingFilename(state.briefing, "carousel-copy", "json")); +} + +async function copyBriefingEvidenceLink(){ + if (state.briefing?.status !== "ready" || !els.briefingCopyLink) return; + const href = new URL(state.briefing.evidence_entry_point || "./?tab=semiconductors", window.location.href).href; + const copied = await copyTextToClipboard(href); + els.briefingCopyLink.textContent = copied ? "Evidence link copied" : "Link ready in address bar"; + if (!copied) window.history.replaceState(null, "", href); + setTimeout(() => { els.briefingCopyLink.textContent = "Copy evidence link"; }, 1600); +} + function renderChipMonthly(){ if (!els.chipMonthlySignals) return; const file = normalizeISO3(state.semiconductorMonthlyFile?.reporter_iso3) === state.chipCountry ? state.semiconductorMonthlyFile : null; @@ -2675,7 +2755,8 @@ function renderSemiconductorAtlas(){ renderChipDistribution(summary); renderChipCountry(summary); renderChipPublicationChanges(); - renderChipMonthly(); + renderDistributionBriefing(); + renderChipMonthly(); renderChipTimeline(reference, sources); renderChipCapacitySignals(reference, sources); renderChipSources(reference); @@ -2996,7 +3077,7 @@ async function main(){ console.warn("[TradeGravity] iso3_to_iso2.json not loaded, using fallback map.", err); } - const [res, metaRes, seriesRes, qualityRes, productIndexRes, strategicIndexRes, tariffIndexRes, matrixIndexRes, mirrorIndexRes, catalogRes, semiconductorReferenceRes, semiconductorMonthlyIndexRes, publicationChangesRes] = await Promise.all([ + const [res, metaRes, seriesRes, qualityRes, productIndexRes, strategicIndexRes, tariffIndexRes, matrixIndexRes, mirrorIndexRes, catalogRes, semiconductorReferenceRes, semiconductorMonthlyIndexRes, publicationChangesRes, briefingRes] = await Promise.all([ fetch(DATA_URL, { cache: "no-store" }), fetch(META_URL, { cache: "no-store" }).catch(() => null), fetch(SERIES_URL, { cache: "no-store" }).catch(() => null), @@ -3010,6 +3091,7 @@ async function main(){ fetch(SEMICONDUCTOR_REFERENCE_URL, { cache: "no-store" }).catch(() => null), fetch(SEMICONDUCTOR_MONTHLY_INDEX_URL, { cache: "no-store" }).catch(() => null), fetch(PUBLICATION_CHANGES_URL, { cache: "no-store" }).catch(() => null), + fetch(BRIEFING_URL, { cache: "no-store" }).catch(() => null), ]); if (!res.ok) throw new Error(`Dataset request failed (${res.status})`); const data = await res.json(); @@ -3028,6 +3110,7 @@ async function main(){ const semiconductorReference = semiconductorReferenceRes?.ok ? await semiconductorReferenceRes.json().catch(() => null) : null; const semiconductorMonthlyIndex = semiconductorMonthlyIndexRes?.ok ? await semiconductorMonthlyIndexRes.json().catch(() => null) : null; const publicationChanges = publicationChangesRes?.ok ? await publicationChangesRes.json().catch(() => null) : null; + const briefing = briefingRes?.ok ? normalizeBriefing(await briefingRes.json().catch(() => null)) : null; state.generatedAt = data.generated_at || data.generatedAt || "-"; state.schemaVersion = String(metadata?.schema_version || data.schema_version || ""); @@ -3044,6 +3127,7 @@ async function main(){ state.semiconductorReference = semiconductorReference; state.semiconductorMonthlyIndex = semiconductorMonthlyIndex; state.publicationChanges = publicationChanges; + state.briefing = briefing; state.meta = metadata; state.resourceStates = [ { label: "metadata", ready: Boolean(metadata) }, @@ -3058,6 +3142,7 @@ async function main(){ { label: "semiconductor atlas", ready: Boolean(semiconductorReference) }, { label: "monthly semiconductor index", ready: Boolean(semiconductorMonthlyIndex) }, { label: "publication change feed", ready: Boolean(publicationChanges) }, + { label: "distribution briefing", ready: Boolean(briefing) }, ]; const initialView = parseViewState(window.location.search); state.metric = initialView.metric; @@ -3245,6 +3330,9 @@ async function main(){ els.chipRoleLandscape?.addEventListener("click", handleChipNavigation); els.chipDistribution?.addEventListener("click", handleChipNavigation); els.chipDownloadCSV?.addEventListener("click", downloadChipCSV); + els.briefingDownloadEmail?.addEventListener("click", downloadBriefingEmail); + els.briefingDownloadCarousel?.addEventListener("click", downloadBriefingCarousel); + els.briefingCopyLink?.addEventListener("click", copyBriefingEvidenceLink); els.chipScenarioForm?.addEventListener("submit", event => { event.preventDefault(); runChipScenario(); diff --git a/site/briefing-tools.js b/site/briefing-tools.js new file mode 100644 index 0000000..f9caa77 --- /dev/null +++ b/site/briefing-tools.js @@ -0,0 +1,62 @@ +(function attachBriefingTools(global) { + "use strict"; + + const READY_SIGNAL_KINDS = ["reporter_total_change", "anchor_share_shift", "product_total_change"]; + const READY_SLIDE_ROLES = ["cover", "scale", "anchor_balance", "product", "method", "cta"]; + + function normalizeBriefing(candidate) { + if (!candidate || candidate.schema_version !== "1.0") return null; + if (!['ready', 'unavailable'].includes(candidate.status)) return null; + if (typeof candidate.edition_id !== "string" || typeof candidate.generated_at !== "string") return null; + if (candidate.review_required !== true || candidate.email?.send_policy !== "manual_review_required") return null; + if (candidate.social_carousel?.review_policy !== "manual_review_required" || candidate.social_carousel?.aspect_ratio !== "4:5") return null; + const signals = Array.isArray(candidate.signals) ? candidate.signals : []; + const slides = Array.isArray(candidate.social_carousel?.slides) ? candidate.social_carousel.slides : []; + if (candidate.status === "unavailable") { + return signals.length === 0 && slides.length === 0 ? candidate : null; + } + if (signals.length !== READY_SIGNAL_KINDS.length || slides.length !== READY_SLIDE_ROLES.length) return null; + if (!READY_SIGNAL_KINDS.every((kind, index) => signals[index]?.kind === kind)) return null; + if (!READY_SLIDE_ROLES.every((role, index) => slides[index]?.role === role && slides[index]?.order === index + 1)) return null; + if (!signals.every(signal => typeof signal.id === "string" && typeof signal.title === "string" && typeof signal.summary === "string" && Array.isArray(signal.evidence) && signal.evidence.length >= 2)) return null; + if (!slides.every(slide => typeof slide.headline === "string" && typeof slide.body === "string" && Array.isArray(slide.evidence) && slide.evidence.length > 0)) return null; + return candidate; + } + + function materializeEmailMarkdown(briefing, baseURL) { + const normalized = normalizeBriefing(briefing); + if (!normalized || typeof normalized.email?.markdown !== "string") return ""; + const base = String(baseURL || "").trim().replace(/\/+$/, ""); + if (!/^https?:\/\//i.test(base)) return normalized.email.markdown; + return normalized.email.markdown.replaceAll("{{BASE_URL}}", base); + } + + function buildCarouselBundle(briefing, evidenceBaseURL) { + const normalized = normalizeBriefing(briefing); + if (!normalized || normalized.status !== "ready") return null; + return { + schema_version: "1.0", + generated_at: normalized.generated_at, + edition_id: normalized.edition_id, + review_required: true, + evidence_base_url: String(evidenceBaseURL || ""), + social_carousel: normalized.social_carousel, + caveats: Array.isArray(normalized.caveats) ? normalized.caveats : [], + }; + } + + function briefingFilename(briefing, suffix, extension) { + const edition = typeof briefing?.edition_id === "string" ? briefing.edition_id : "tradegravity-briefing"; + const safeEdition = edition.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "") || "tradegravity-briefing"; + const safeSuffix = String(suffix || "draft").toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "") || "draft"; + const safeExtension = String(extension || "txt").toLowerCase().replace(/[^a-z0-9]+/g, "") || "txt"; + return `${safeEdition}-${safeSuffix}.${safeExtension}`; + } + + global.TradeGravityBriefingTools = Object.freeze({ + normalizeBriefing, + materializeEmailMarkdown, + buildCarouselBundle, + briefingFilename, + }); +})(globalThis); diff --git a/site/briefing-tools.test.cjs b/site/briefing-tools.test.cjs new file mode 100644 index 0000000..ad8367f --- /dev/null +++ b/site/briefing-tools.test.cjs @@ -0,0 +1,60 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); +const vm = require("node:vm"); + +const source = fs.readFileSync(path.join(__dirname, "briefing-tools.js"), "utf8"); +const context = { globalThis: {} }; +vm.createContext(context); +vm.runInContext(source, context); +const tools = context.globalThis.TradeGravityBriefingTools; + +function readyBriefing() { + const signals = ["reporter_total_change", "anchor_share_shift", "product_total_change"].map((kind, index) => ({ + id: `signal-${index}`, + kind, + title: `Signal ${index}`, + summary: "Summary", + evidence: ["./one.json", "./two.json"], + })); + const roles = ["cover", "scale", "anchor_balance", "product", "method", "cta"]; + return { + schema_version: "1.0", + generated_at: "2026-07-17T00:00:00Z", + edition_id: "semiconductor-pulse-2026-05-20260717T000000Z", + status: "ready", + review_required: true, + signals, + email: { send_policy: "manual_review_required", markdown: "[Evidence]({{BASE_URL}}?tab=semiconductors)" }, + social_carousel: { + aspect_ratio: "4:5", + review_policy: "manual_review_required", + slides: roles.map((role, index) => ({ order: index + 1, role, headline: "Headline", body: "Body", evidence: ["./one.json"] })), + }, + caveats: ["Caveat"], + }; +} + +test("normalizes only review-gated briefing contracts", () => { + const briefing = readyBriefing(); + assert.equal(tools.normalizeBriefing(briefing), briefing); + briefing.email.send_policy = "automatic"; + assert.equal(tools.normalizeBriefing(briefing), null); +}); + +test("materializes the public evidence URL without changing canonical copy", () => { + const briefing = readyBriefing(); + const draft = tools.materializeEmailMarkdown(briefing, "https://example.test/TradeGravity/"); + assert.equal(draft, "[Evidence](https://example.test/TradeGravity?tab=semiconductors)"); + assert.match(briefing.email.markdown, /\{\{BASE_URL\}\}/); +}); + +test("builds a cited carousel bundle and a filesystem-safe filename", () => { + const briefing = readyBriefing(); + const bundle = tools.buildCarouselBundle(briefing, "https://example.test/TradeGravity/data/"); + assert.equal(bundle.review_required, true); + assert.equal(bundle.social_carousel.slides.length, 6); + assert.equal(bundle.evidence_base_url, "https://example.test/TradeGravity/data/"); + assert.equal(tools.briefingFilename(briefing, "Instagram draft", "JSON"), "semiconductor-pulse-2026-05-20260717t000000z-instagram-draft.json"); +}); diff --git a/site/index.html b/site/index.html index 91691bf..3b66f7b 100644 --- a/site/index.html +++ b/site/index.html @@ -15,6 +15,7 @@ + @@ -262,6 +263,20 @@

US–China Chip Supply Chain Lens

Loading the publish-to-publish comparison…
+
+
+
Distribution desk · reviewed drafts

Email brief and social carousel

+ Loading briefing… +
+
Loading the deterministic distribution brief…
+
+ + + +
+

Delivery is not configured. This static build creates manually reviewed drafts and does not collect subscriber data or publish directly to social platforms.

+
+
Current structural shifts

What is changing

External context · dated sources
Loading current semiconductor trends…
diff --git a/site/structure.test.cjs b/site/structure.test.cjs index 6e46672..aa3e331 100644 --- a/site/structure.test.cjs +++ b/site/structure.test.cjs @@ -15,9 +15,10 @@ test("index loads trusted helpers before the application and keeps D3 pinned", ( const semiconductorToolsIndex = html.indexOf('src="./semiconductor-tools.js"'); const experienceToolsIndex = html.indexOf('src="./experience-tools.js"'); const newsToolsIndex = html.indexOf('src="./news-tools.js"'); + const briefingToolsIndex = html.indexOf('src="./briefing-tools.js"'); const d3Index = html.indexOf('src="https://cdn.jsdelivr.net/npm/d3@7.9.0/dist/d3.min.js"'); const appIndex = html.indexOf('src="./app.js"'); - assert.ok(securityIndex >= 0 && dataToolsIndex > securityIndex && explorerToolsIndex > dataToolsIndex && intelligenceToolsIndex > explorerToolsIndex && semiconductorToolsIndex > intelligenceToolsIndex && experienceToolsIndex > semiconductorToolsIndex && newsToolsIndex > experienceToolsIndex && d3Index > newsToolsIndex && appIndex > d3Index); + assert.ok(securityIndex >= 0 && dataToolsIndex > securityIndex && explorerToolsIndex > dataToolsIndex && intelligenceToolsIndex > explorerToolsIndex && semiconductorToolsIndex > intelligenceToolsIndex && experienceToolsIndex > semiconductorToolsIndex && newsToolsIndex > experienceToolsIndex && briefingToolsIndex > newsToolsIndex && d3Index > briefingToolsIndex && appIndex > d3Index); assert.match(html, /integrity="sha384-[A-Za-z0-9+/=]+"/); assert.match(html, /Content-Security-Policy/); }); @@ -105,6 +106,7 @@ test("chip lens exposes coverage, stages, roles, monthly signals, policy, eviden "chipPublicationChanges", "chipTrends", "chipValueChain", "chipRoleLandscape", "chipDistribution", "chipCountryProfile", "chipTimeline", "chipCapacitySignals", "chipMonthlySignals", + "chipBriefingStatus", "chipBriefingSignals", "briefingDownloadEmail", "briefingDownloadCarousel", "briefingCopyLink", "briefingDeliveryNote", "chipScenarioForm", "chipDisruption", "chipSubstitution", "chipScenarioBaseline", "chipScenarioResult", "chipSources", "chipCaveats", ]) { @@ -117,10 +119,15 @@ test("chip lens exposes coverage, stages, roles, monthly signals, policy, eviden assert.match(app, /async function loadSelectedChipMonthly/); assert.match(app, /async function renderMirrorDiagnostics/); assert.match(app, /function runChipScenario/); + assert.match(app, /function renderDistributionBriefing/); + assert.match(app, /function downloadBriefingEmail/); + assert.match(app, /function downloadBriefingCarousel/); assert.match(css, /\.chipValueChain/); + assert.match(css, /\.briefingSignalGrid/); assert.match(css, /\.policyTimeline/); assert.match(css, /\.chipRoleTable/); assert.match(html, /announcements are never counted as operating capacity/i); + assert.match(html, /does not collect subscriber data or publish directly to social platforms/i); }); test("overview treemaps survive hidden-tab resizes and redraw when shown", () => { diff --git a/site/styles.css b/site/styles.css index 9549687..2ee5338 100644 --- a/site/styles.css +++ b/site/styles.css @@ -690,6 +690,15 @@ select option{background:#fff;color:#111} .chipSectionHeading{display:flex;align-items:flex-start;justify-content:space-between;gap:16px;margin-bottom:14px} .chipSectionHeading h2{margin:3px 0 0;font-size:17px} .chipSectionHeading .analysisNote{max-width:520px;margin:0;text-align:right} +.briefingDesk{background:linear-gradient(135deg,rgba(90,162,255,.055),rgba(231,211,124,.03) 55%,rgba(255,255,255,.015))} +.briefingSignalGrid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:9px} +.briefingSignalCard{display:flex;min-width:0;min-height:142px;flex-direction:column;padding:14px;border:1px solid var(--border);border-radius:11px;background:rgba(255,255,255,.018)} +.briefingSignalCard > span{color:var(--hi);font:700 9px var(--mono);text-transform:uppercase} +.briefingSignalCard h3{margin:9px 0 6px;font-size:12px;line-height:1.35} +.briefingSignalCard p{margin:0;color:var(--muted);font-size:10px;line-height:1.5} +.briefingSignalCard small{margin-top:auto;padding-top:12px;color:var(--muted2);font:8px var(--mono);line-height:1.4} +.briefingActions{display:flex;flex-wrap:wrap;gap:8px;margin-top:12px;padding-top:12px;border-top:1px solid var(--border)} +.briefingActions .btn:disabled{cursor:not-allowed;opacity:.45} .chipTrendGrid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:9px} .chipTrendCard{min-width:0;padding:14px;border:1px solid var(--border);border-radius:11px;background:linear-gradient(145deg,rgba(90,162,255,.055),rgba(255,255,255,.015))} .chipTrendCard > span{color:var(--hi);font:9px var(--mono)} @@ -782,7 +791,7 @@ select option{background:#fff;color:#111} .modelRoadmap{grid-template-columns:repeat(2,minmax(0,1fr))} .chipControlBar{grid-template-columns:repeat(2,minmax(0,1fr))} .chipCoverageSummary{grid-column:1/-1} - .chipTrendGrid{grid-template-columns:repeat(2,minmax(0,1fr))} + .chipTrendGrid,.briefingSignalGrid{grid-template-columns:repeat(2,minmax(0,1fr))} } @media (max-width: 650px){ .tabBar{padding:0 10px} @@ -793,7 +802,7 @@ select option{background:#fff;color:#111} .chipControlBar{grid-template-columns:1fr} .chipAnalysisGrid{padding:10px 10px 0} .chipEvidenceGrid{padding-bottom:10px} - .chipTrendGrid,.sourceRegister,.chipScenarioForm{grid-template-columns:1fr} + .chipTrendGrid,.briefingSignalGrid,.sourceRegister,.chipScenarioForm{grid-template-columns:1fr} .chipTrendCard p{min-height:0} .chipSignalMetrics{grid-template-columns:repeat(2,minmax(0,1fr))} .chipSectionHeading{flex-direction:column}.chipSectionHeading .analysisNote{text-align:left}