diff --git a/.github/workflows/build-distribution-kit.yml b/.github/workflows/build-distribution-kit.yml new file mode 100644 index 0000000..6fe9405 --- /dev/null +++ b/.github/workflows/build-distribution-kit.yml @@ -0,0 +1,68 @@ +name: Build reviewed distribution kit + +on: + workflow_dispatch: + inputs: + theme: + description: Native carousel theme + required: true + default: intelligence-dark + type: choice + options: + - intelligence-dark + - editorial-light + +permissions: + contents: read + +concurrency: + group: distribution-kit-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check out source + uses: actions/checkout@v7 + + - name: Check out published data + uses: actions/checkout@v7 + with: + ref: gh-pages + path: published + + - uses: actions/setup-go@v6 + with: + go-version: "1.25.12" + cache: true + + - name: Validate the published evidence bundle + run: go run ./cmd/validator -dir published/data -min-reporters 40 + + - name: Build the review-pending kit + run: | + go run ./cmd/distributor \ + -briefing published/data/briefing.json \ + -out "${RUNNER_TEMP}/distribution-kit" \ + -base-url https://elecpapaya.github.io/TradeGravity/ \ + -theme "${{ inputs.theme }}" + + - name: Verify upload assets remain review-gated + shell: bash + run: | + test -f "${RUNNER_TEMP}/distribution-kit/carousel/slide-01.png" + test -f "${RUNNER_TEMP}/distribution-kit/carousel/slide-06.png" + test -f "${RUNNER_TEMP}/distribution-kit/carousel/caption.md" + grep -q '"distribution_status": "review_pending"' "${RUNNER_TEMP}/distribution-kit/manifest.json" + grep -q '"social_publish_authorized": false' "${RUNNER_TEMP}/distribution-kit/manifest.json" + grep -q '"theme": "${{ inputs.theme }}"' "${RUNNER_TEMP}/distribution-kit/manifest.json" + grep -q '"caption_path": "carousel/caption.md"' "${RUNNER_TEMP}/distribution-kit/manifest.json" + + - name: Upload review artifact + uses: actions/upload-artifact@v4 + with: + name: tradegravity-distribution-kit-${{ github.run_id }} + path: ${{ runner.temp }}/distribution-kit + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 2eddbd3..76fd2f5 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -45,11 +45,43 @@ jobs: - name: Validate offline sample dataset run: go run ./cmd/validator -dir examples/sample-data -min-reporters 3 + - name: Build offline distribution kit + run: | + go run ./cmd/distributor \ + -briefing examples/sample-data/briefing.json \ + -out "${RUNNER_TEMP}/distribution-kit" \ + -base-url https://example.org/TradeGravity/ + test -f "${RUNNER_TEMP}/distribution-kit/email/body.html" + grep -q '{{UNSUBSCRIBE_URL}}' "${RUNNER_TEMP}/distribution-kit/email/body.html" + test -f "${RUNNER_TEMP}/distribution-kit/carousel/slide-06.svg" + test -f "${RUNNER_TEMP}/distribution-kit/carousel/slide-06.png" + test -f "${RUNNER_TEMP}/distribution-kit/carousel/caption.md" + grep -q 'https://example.org/TradeGravity/?tab=semiconductors' "${RUNNER_TEMP}/distribution-kit/carousel/caption.md" + test -f "${RUNNER_TEMP}/distribution-kit/manifest.json" + grep -q '"theme": "intelligence-dark"' "${RUNNER_TEMP}/distribution-kit/manifest.json" + grep -q '"caption_path": "carousel/caption.md"' "${RUNNER_TEMP}/distribution-kit/manifest.json" + go run ./cmd/distribution-approval \ + -kit "${RUNNER_TEMP}/distribution-kit" \ + -reviewer ci-synthetic-reviewer \ + -audience synthetic-internal-pilot \ + -channels email,instagram \ + -approved-at 2026-07-17T12:00:00Z \ + -attest-reviewed + test -f "${RUNNER_TEMP}/distribution-kit/approval.json" + grep -q '"provider_delivery_ready": false' "${RUNNER_TEMP}/distribution-kit/approval.json" + go run ./cmd/instagram-preflight \ + -kit "${RUNNER_TEMP}/distribution-kit" \ + -out "${RUNNER_TEMP}/instagram-preflight.json" \ + -generated-at 2026-07-17T13:00:00Z + grep -q '"manual_upload_required": true' "${RUNNER_TEMP}/instagram-preflight.json" + grep -q '"automatic_publish_authorized": false' "${RUNNER_TEMP}/instagram-preflight.json" + grep -q '"contains_credentials": false' "${RUNNER_TEMP}/instagram-preflight.json" + - name: Check browser JavaScript - run: node --check site/app.js && node --check site/security.js && node --check site/data-tools.js && node --check site/explorer-tools.js && node --check site/intelligence-tools.js && node --check site/semiconductor-tools.js && node --check site/experience-tools.js && node --check site/news-tools.js + run: node --check site/app.js && node --check site/security.js && node --check site/data-tools.js && node --check site/explorer-tools.js && node --check site/intelligence-tools.js && node --check site/semiconductor-tools.js && node --check site/experience-tools.js && node --check site/news-tools.js && node --check site/briefing-tools.js - name: Test browser data helpers - run: node --test site/security.test.cjs site/data-tools.test.cjs site/explorer-tools.test.cjs site/intelligence-tools.test.cjs site/semiconductor-tools.test.cjs site/experience-tools.test.cjs site/news-tools.test.cjs site/structure.test.cjs + run: node --test site/security.test.cjs site/data-tools.test.cjs site/explorer-tools.test.cjs site/intelligence-tools.test.cjs site/semiconductor-tools.test.cjs site/experience-tools.test.cjs site/news-tools.test.cjs site/briefing-tools.test.cjs site/structure.test.cjs - name: Validate citation metadata run: | diff --git a/.gitignore b/.gitignore index 312d1f3..f080e75 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,13 @@ /site/data/ tradegravity.db + +/private/ +/distribution-kit/ +/delivery-preflight.json +/instagram-preflight.json +/email-launch-authorization.json +subscriptions.db +subscriptions.db-* +delivery-ledger.db +delivery-ledger.db-* diff --git a/CHANGELOG.md b/CHANGELOG.md index c41676f..332f8e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,16 @@ All notable changes to TradeGravity will be documented in this file. The project - Global PNG snapshot, CSV, and Markdown summary-report exports for the active analysis view. - Always-visible metric, observation-period, scope, and limitation context plus a definitions dialog. - Current, partial, and degraded publication-health states with recovery guidance. +- A validated `briefing.json` contract that derives three cited semiconductor observations and exposes review-gated email Markdown and 4:5 social-carousel copy without collecting subscribers or publishing automatically. +- An offline distribution-kit CLI and manual read-only Actions workflow that render one-primary-CTA email HTML with an unsubscribe placeholder, six matched 1080×1350 SVG originals and PNG upload assets, alt text, approval gates, and deterministic file hashes without sending or posting. +- A second original `editorial-light` native carousel theme behind the same validated renderer interface; theme choice is recorded in the manifest and therefore bound to editorial approval, with no browser runtime, remote font, image fetch, or arbitrary HTML input. +- Automated palette contrast checks across every gradient stop, including normal text and large bold role labels; the light theme's muted and accent colors were tightened to retain the documented floor. +- A review-pending Instagram `caption.md` derived from the same three validated signals, with comparison period, evidence link, conservative scope note, restrained tags, an editorial length ceiling, and manifest/approval tamper protection. +- An Instagram manual-publish preflight CLI that requires channel approval, verifies every PNG plus caption and alt-text contracts, emits content-free aggregate evidence outside the kit, and explicitly carries no credentials or publish authorization. +- A content-release approval CLI that rejects changed, missing, or untracked kit files and binds the verified manifest to a reviewer, audience label, time, and email/Instagram channel set without claiming delivery readiness. +- A local email preflight CLI that validates double opt-in, audience identity, suppression precedence, timestamps, duplicate addresses, unique opaque HTTPS unsubscribe URLs, and a pilot ceiling while emitting an aggregate plan with no recipient addresses or tokens and no delivery authorization. +- A private SQLite subscription-registry CLI and subscription HTTP service with a default-off double-opt-in signup form, short-lived purpose-separated HMAC confirmation links, read-only scanner-safe GET plus explicit confirmation POST, stable Resend confirmation idempotency, form-encoded one-click unsubscribe, signed raw-body provider feedback, durable global suppression, security headers, and private preflight exports. +- A short-lived email launch-approval contract and Resend pilot CLI that replay consent/suppression checks at send time, render recipient-specific visible and RFC one-click unsubscribe links, use one recipient and one provider idempotency key per request, and prevent automatic duplicate or uncertain retries with a private HMAC-keyed SQLite delivery ledger. A separate reconciliation CLI records provider-confirmed acceptance or non-acceptance without storing recipient PII; only the latter plus a different launch-authorization digest permits retry. ### Changed diff --git a/README.md b/README.md index fb4629b..a46f7ba 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,11 @@ TradeGravity is an open-source pipeline and static intelligence dashboard for un - **Live demo:** https://elecpapaya.github.io/TradeGravity/ - **System design:** [DESIGN.md](DESIGN.md) - **Published data schema:** [docs/DATA_SCHEMA.md](docs/DATA_SCHEMA.md) +- **Reviewed distribution workflow:** [docs/DISTRIBUTION.md](docs/DISTRIBUTION.md) +- **Email consent and suppression preflight:** [docs/EMAIL_DELIVERY_PREFLIGHT.md](docs/EMAIL_DELIVERY_PREFLIGHT.md) +- **Provider-backed email pilot:** [docs/EMAIL_PROVIDER_PILOT.md](docs/EMAIL_PROVIDER_PILOT.md) +- **Private registry and unsubscribe service:** [docs/UNSUBSCRIBE_SERVICE.md](docs/UNSUBSCRIBE_SERVICE.md) +- **Instagram manual-publish preflight:** [docs/INSTAGRAM_PREFLIGHT.md](docs/INSTAGRAM_PREFLIGHT.md) - **Semiconductor atlas methodology:** [docs/SEMICONDUCTOR_ATLAS.md](docs/SEMICONDUCTOR_ATLAS.md) - **Reuse examples:** [docs/USAGE.md](docs/USAGE.md) - **Data rights and attribution:** [docs/DATA_RIGHTS.md](docs/DATA_RIGHTS.md) @@ -70,6 +75,13 @@ The pipeline refresh timestamp indicates when TradeGravity generated the site; i - HS2 product mix for the selected reporter, kept separate from WITS headline totals. - Shareable Overview, US–China Lens, Chip Lens, Products, Data & Quality, and Scenario Lab tabs with synchronized filters, country, semiconductor stage/context, product, tariff, and scenario-assumption state. - A semiconductor Pulse that separates latest month-to-month movement from publish-to-publish coverage and value revisions, with a machine-readable bounded change feed. +- A deterministic `briefing.json` distribution draft with three cited semiconductor observations, review-gated email Markdown, and review-gated 4:5 carousel copy. The static site does not collect subscribers, send email, or publish to social platforms. +- An offline `cmd/distributor` build that turns a ready briefing into email HTML/Markdown, a cited Instagram caption, alt text, and six matched 1080×1350 SVG/PNG cards in either `intelligence-dark` or `editorial-light`; all assets share review gates and deterministic hashes without making a network request. +- An aggregate-only `cmd/instagram-preflight` that requires an unchanged Instagram approval, decodes all six PNGs, validates caption evidence/scope/tags and six alt-text sections, refuses output inside the kit, and keeps credentials and automatic publishing explicitly false. +- A fail-closed `cmd/distribution-approval` step that verifies the complete file set and SHA-256 manifest before recording a channel-specific content approval; provider delivery, subscriber consent, and automatic publishing remain explicitly false. +- A local `cmd/distribution-preflight` gate that validates private double-opt-in and suppression CSVs, approved-audience identity, and unique opaque HTTPS unsubscribe URLs, enforces a pilot ceiling, and writes only aggregate counts and digests—never recipient addresses or tokens—while keeping provider configuration and delivery authorization false. +- A separate SQLite `cmd/subscription-registry` and bounded `cmd/unsubscribe-service` that can collect double-opt-in consent outside the static dashboard, send short-lived Resend confirmations, activate only on explicit confirmation POST, issue HMAC-authenticated links without email/audience claims, keep link-scanner GETs read-only, record idempotent RFC one-click suppressions, verify signed Resend feedback, and export private preflight inputs. +- A fail-closed Resend pilot path that binds a one-hour launch approval to the exact aggregate preflight, sender, audience, and content/input digests; reruns the consent and suppression checks immediately before delivery; adds visible and header one-click unsubscribe links; isolates every recipient in a separate provider request; and records only HMAC recipient keys in a private SQLite ledger. Accepted or uncertain attempts are never sent again automatically; provider-confirmed non-acceptance still requires a recorded reconciliation and a new launch approval. - Two-anchor position metrics whose formulas are visible: USA share, China share, exposure balance, position shift, dual exposure, and anchor-growth divergence. - Unadjusted bilateral mirror-reporting diagnostics that compare both countries' reports without choosing either as ground truth or treating the difference as fraud, evasion, rerouting, or an adjusted estimate. - An illustrative HS6 tariff sensitivity lab that can load a published MFN rate and product import baseline while exposing elasticity, pass-through, fallback, and source assumptions. @@ -141,6 +153,7 @@ The public deployment exposes stable machine-readable endpoints: - `https://elecpapaya.github.io/TradeGravity/data/semiconductors/reference.json` - `https://elecpapaya.github.io/TradeGravity/data/semiconductors/monthly/index.json` - `https://elecpapaya.github.io/TradeGravity/data/changes.json` +- `https://elecpapaya.github.io/TradeGravity/data/briefing.json` - `https://elecpapaya.github.io/TradeGravity/data/tariffs/index.json` - `https://elecpapaya.github.io/TradeGravity/data/bilateral-matrix/index.json` - `https://elecpapaya.github.io/TradeGravity/data/mirror/index.json` @@ -276,13 +289,13 @@ This repository reads operating-system environment variables and does not load a ## Generated files and deployment - Local SQLite database: `tradegravity.db` -- Published JSON: `meta.json`, `catalog.json`, `changes.json`, `latest.json`, `series.json`, `quality.json`, `context.json`, `products/`, `strategic-hs6/`, `semiconductors/reference.json`, `semiconductors/monthly/`, `tariffs/`, `bilateral-matrix/`, `mirror/`, and `explanations/` under `site/data/` +- Published JSON: `meta.json`, `catalog.json`, `changes.json`, `briefing.json`, `latest.json`, `series.json`, `quality.json`, `context.json`, `products/`, `strategic-hs6/`, `semiconductors/reference.json`, `semiconductors/monthly/`, `tariffs/`, `bilateral-matrix/`, `mirror/`, and `explanations/` under `site/data/` Generated data and the local database are intentionally not committed to the default branch. The scheduled or manually dispatched core workflow runs the broad collectors and saves its validated database as a three-day Actions artifact. The staggered semiconductor workflow restores that artifact and the previous `gh-pages` publication, adds annual and monthly chip observations for [`configs/chip_connectors.csv`](configs/chip_connectors.csv), emits a validated publish-to-publish `changes.json`, and deploys `site/` to the `gh-pages` branch. A `main` push uses the latest validated `data/` directory from `gh-pages` and redeploys the site without calling WITS, UN Comtrade, WITS/TRAINS, or World Bank APIs. This keeps code-only deployments fast while the weekly refresh remains the source of new published observations. The fast deployment intentionally fails if `gh-pages` does not contain `data/latest.json` and `data/meta.json`. Bootstrap or repair the published dataset by manually running **Update TradeGravity core**, then **Update TradeGravity semiconductor**; the second workflow waits out any remaining quota window before it publishes. -Before deployment, `cmd/validator` checks provenance across every artifact, reporter uniqueness, periods, non-negative finite values, totals and shares, matrix availability/count identities, tariff rate identities, product keys, bounded publication-change arithmetic and ordering, context coverage, and explanation evidence references. +Before deployment, `cmd/validator` checks provenance across every artifact, reporter uniqueness, periods, non-negative finite values, totals and shares, matrix availability/count identities, tariff rate identities, product keys, bounded publication-change arithmetic and ordering, briefing arithmetic and mandatory human-review gates, context coverage, and explanation evidence references. ## Maintenance and contributing diff --git a/ROADMAP.md b/ROADMAP.md index ad3f1f1..0729f00 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -38,6 +38,13 @@ TradeGravity is maintained as a small, inspectable public-data pipeline with an - Publish USA/China exposure balance, position shift, and dual-exposure metrics with explicit formulas and non-alignment caveats across the Intelligence and Chip Lens views. - Collect and publish a focused 12-month, 30-code semiconductor turning-point panel for selected connector economies. - Publish a validated semiconductor Pulse with latest-month movement, previous-publication coverage/value changes, a bounded machine-readable change feed, and evidence endpoints in the Markdown report. +- Publish a deterministic, validator-gated semiconductor distribution brief with cited email Markdown and 4:5 carousel copy; require human review and keep delivery and subscriber storage disabled in the static site. +- Generate a deterministic offline review kit with two network-free native carousel themes, one-primary-CTA email HTML/Markdown, a cited and scope-limited Instagram caption, six matched 1080×1350 SVG/PNG assets, alt text, an editorial/privacy checklist, and a theme/caption-bound SHA-256 manifest; expose it through a read-only manual Actions workflow. +- Require an aggregate Instagram preflight after content approval to revalidate manifest integrity, six upload-ready PNG dimensions, caption evidence/scope/tags, and alt-text completeness while retaining manual upload and false automatic-publish authorization. +- Bind each reviewed kit to an immutable manifest digest, reviewer, non-sensitive audience label, explicit time, and approved channel set while keeping consent, provider-delivery, and automatic-publish readiness false. +- Validate private double-opt-in and suppression CSVs plus unique opaque HTTPS unsubscribe URLs against an email-approved kit, enforce a bounded pilot audience, and emit an address/token-free aggregate preflight plan with provider and delivery authorization still false. +- Maintain consent and suppression in a private SQLite registry; provide an opt-in signup form with short-lived, purpose-separated HMAC confirmation links and stable Resend idempotency; require explicit confirmation POST; issue PII-free unsubscribe links; and verify/deduplicate signed Resend suppression feedback behind an HTTPS reverse proxy. +- Provide a fail-closed Resend pilot adapter with short-lived launch approval, live preflight replay, per-recipient requests, RFC one-click headers, provider idempotency keys, a PII-free local delivery ledger, and explicit provider-evidence reconciliation. Require a new launch approval after confirmed non-acceptance and never automatically retry an unresolved request; keep credentials, subscriber inputs, authorization, and the ledger outside the repository. - Publish unadjusted mirror-reporting diagnostics against USA and China counterpart reports without selecting a ground truth or claiming fraud, rerouting, or reconciliation. - Register only free/public semiconductor evidence layers, including OECD ICIO as lagged industry context, and validate that paid/proprietary sources cannot become required metric inputs. @@ -48,6 +55,7 @@ TradeGravity is maintained as a small, inspectable public-data pipeline with an - Run the documented task with at least three students, researchers, or developers and publish consented findings through [the public study tracker](https://github.com/elecpapaya/TradeGravity/issues/3). - Track task completion, interpretation errors, and time-to-answer in [`docs/USER_TESTING.md`](docs/USER_TESTING.md); never substitute synthetic sessions for real participants. - Publish and maintain the reproducible ASEAN example notebook. +- Verify the deployed `briefing.json`, then run a small editorial review that checks whether readers can trace every distributed observation back to its period, value, and evidence artifact. ## Next: usability and analytical depth @@ -62,11 +70,13 @@ TradeGravity is maintained as a small, inspectable public-data pipeline with an - Prototype ESI/ECI/ICI/SPDI/RPI only after formulas, benchmark datasets, uncertainty, and validation cases are documented. - Add tariff-change decomposition and Marimekko/waterfall views when the required multi-year product coverage is reliable. - [Add versioned scenario manifests and reproducible result artifacts](https://github.com/elecpapaya/TradeGravity/issues/13) before introducing SMART-like substitution or welfare outputs. +- Run the first provider-backed email pilot only after configuring Resend and an authenticated sender domain, deploying signup/confirmation/unsubscribe/signed-feedback endpoints over HTTPS with proxy rate limits, proving feedback replay and restore operations, and obtaining a fresh launch approval for a deliberately confirmed audience. The adapter exists, but no production send is implied. Do not place provider credentials or subscriber data in the static site. ## Later: provider resilience and interpretation - Add configurable provider fallback without mixing provenance silently. - Extend the implemented semiconductor publication-change feed to headline, tariff, and matrix artifacts after each layer has a stable comparison key and revision policy. - Expand grounded explanations after external evaluation confirms they reduce rather than increase interpretation errors. +- Render branded Instagram/LinkedIn card images from the reviewed carousel contract only after an accessibility, citation legibility, platform-rights, and editorial-approval review; keep direct social publishing as a separate, revocable step. Priorities may change when upstream APIs change or users report higher-impact needs. Roadmap discussion should happen in a GitHub issue so decisions remain public and reviewable. diff --git a/cmd/distribution-approval/main.go b/cmd/distribution-approval/main.go new file mode 100644 index 0000000..c764178 --- /dev/null +++ b/cmd/distribution-approval/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/distributionkit" +) + +func main() { + kitDir := flag.String("kit", "distribution-kit", "reviewed distribution-kit directory") + reviewer := flag.String("reviewer", "", "human reviewer name or stable handle; do not include subscriber data") + audience := flag.String("audience", "", "non-sensitive audience label, for example internal-pilot") + channelsValue := flag.String("channels", "", "comma-separated content channels: email,instagram") + approvedAtValue := flag.String("approved-at", "", "explicit RFC3339 approval time") + attested := flag.Bool("attest-reviewed", false, "confirm evidence, copy, rights, assets, and alt text were reviewed") + flag.Parse() + + approvedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*approvedAtValue)) + if err != nil { + fatal(fmt.Errorf("approved-at must be RFC3339: %w", err)) + } + channels := splitChannels(*channelsValue) + approval, content, err := distributionkit.Approve(*kitDir, distributionkit.ApprovalRequest{ + Reviewer: *reviewer, + Audience: *audience, + Channels: channels, + ApprovedAt: approvedAt, + Attested: *attested, + }) + if err != nil { + fatal(err) + } + if err := distributionkit.WriteApproval(*kitDir, content); err != nil { + fatal(err) + } + fmt.Printf("content approval recorded (edition=%s channels=%s scope=%s delivery_ready=%t)\n", approval.EditionID, strings.Join(approval.Channels, ","), approval.Scope, approval.ProviderDeliveryReady) +} + +func splitChannels(value string) []string { + if strings.TrimSpace(value) == "" { + return nil + } + parts := strings.Split(value, ",") + for index := range parts { + parts[index] = strings.TrimSpace(parts[index]) + } + return parts +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "distribution approval failed:", err) + os.Exit(1) +} diff --git a/cmd/distribution-preflight/main.go b/cmd/distribution-preflight/main.go new file mode 100644 index 0000000..5246ca3 --- /dev/null +++ b/cmd/distribution-preflight/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/deliverypreflight" +) + +func main() { + kitDir := flag.String("kit", "distribution-kit", "approved distribution-kit directory") + subscribers := flag.String("subscribers", "", "local double-opt-in subscriber CSV outside the kit") + suppressions := flag.String("suppressions", "", "local unsubscribe/bounce/complaint suppression CSV outside the kit") + output := flag.String("out", "delivery-preflight.json", "new aggregate-only preflight JSON outside the kit") + generatedAtValue := flag.String("generated-at", "", "explicit RFC3339 preflight time") + maxRecipients := flag.Int("max-recipients", 25, "fail when the consented unsuppressed pilot audience exceeds this limit (1-1000)") + flag.Parse() + + generatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*generatedAtValue)) + if err != nil { + fatal(fmt.Errorf("generated-at must be RFC3339: %w", err)) + } + result, err := deliverypreflight.Build(deliverypreflight.Request{ + KitDir: *kitDir, + SubscriberCSV: *subscribers, + SuppressionCSV: *suppressions, + GeneratedAt: generatedAt, + MaxRecipients: *maxRecipients, + }) + if err != nil { + fatal(err) + } + if err := deliverypreflight.Write(*output, *kitDir, result.JSON); err != nil { + fatal(err) + } + fmt.Printf("email preflight passed (edition=%s audience=%s consented=%d suppressed=%d eligible=%d delivery_authorized=%t)\n", result.Plan.EditionID, result.Plan.Audience, result.Plan.Counts.Consented, result.Plan.Counts.Suppressed, result.Plan.Counts.Eligible, result.Plan.DeliveryAuthorized) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "distribution preflight failed:", err) + os.Exit(1) +} diff --git a/cmd/distributor/main.go b/cmd/distributor/main.go new file mode 100644 index 0000000..fe2cdd7 --- /dev/null +++ b/cmd/distributor/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "tradegravity/internal/distributionkit" +) + +func main() { + briefingPath := flag.String("briefing", "site/data/briefing.json", "review-gated briefing.json input") + outputDir := flag.String("out", "distribution-kit", "new output directory; existing paths are never overwritten") + baseURL := flag.String("base-url", "https://elecpapaya.github.io/TradeGravity/", "public evidence base URL") + theme := flag.String("theme", distributionkit.ThemeIntelligenceDark, "carousel theme: intelligence-dark or editorial-light") + flag.Parse() + + raw, err := os.ReadFile(*briefingPath) + if err != nil { + fatal(fmt.Errorf("read briefing: %w", err)) + } + bundle, err := distributionkit.BuildWithOptions(raw, *baseURL, distributionkit.BuildOptions{Theme: *theme}) + if err != nil { + fatal(err) + } + if err := distributionkit.Write(*outputDir, bundle); err != nil { + fatal(err) + } + fmt.Printf("distribution kit built (edition=%s theme=%s files=%d status=%s out=%s)\n", bundle.Manifest.EditionID, bundle.Manifest.Carousel.Theme, len(bundle.Files), bundle.Manifest.DistributionStatus, *outputDir) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "distribution kit failed:", err) + os.Exit(1) +} diff --git a/cmd/email-delivery-reconcile/main.go b/cmd/email-delivery-reconcile/main.go new file mode 100644 index 0000000..e399673 --- /dev/null +++ b/cmd/email-delivery-reconcile/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/emaildelivery" +) + +const ledgerSecretEnvironment = "TRADEGRAVITY_DELIVERY_SECRET" + +func main() { + ledgerPath := flag.String("ledger", "", "private SQLite delivery ledger") + edition := flag.String("edition", "", "edition ID from the stopped delivery") + audience := flag.String("audience", "", "non-sensitive audience label") + recipient := flag.String("recipient", "", "private recipient address used only to derive the ledger key") + outcome := flag.String("outcome", "", "provider-confirmed outcome: accepted or not_accepted") + providerMessageID := flag.String("provider-message-id", "", "required only for an accepted outcome") + resolvedBy := flag.String("resolved-by", "", "operator identity recorded in the private ledger") + evidence := flag.String("evidence", "", "non-sensitive provider-dashboard or support evidence label") + resolvedAtValue := flag.String("resolved-at", "", "explicit RFC3339 reconciliation time") + attest := flag.Bool("attest-provider-checked", false, "attest that the provider dashboard or support record was checked") + flag.Parse() + + if !*attest { + fatal(fmt.Errorf("attest-provider-checked is required")) + } + resolvedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*resolvedAtValue)) + if err != nil { + fatal(fmt.Errorf("resolved-at must be RFC3339: %w", err)) + } + ledger, err := emaildelivery.OpenLedger(*ledgerPath, []byte(os.Getenv(ledgerSecretEnvironment))) + if err != nil { + fatal(err) + } + defer ledger.Close() + result, err := ledger.Reconcile(context.Background(), emaildelivery.ReconciliationRequest{ + EditionID: *edition, + Audience: *audience, + Email: *recipient, + Outcome: strings.TrimSpace(*outcome), + ProviderMessageID: *providerMessageID, + ResolvedBy: *resolvedBy, + Evidence: *evidence, + ResolvedAt: resolvedAt, + }) + if err != nil { + fatal(err) + } + fmt.Printf("delivery reconciliation recorded (edition=%s audience=%s outcome=%s changed=%t already_resolved=%t)\n", *edition, *audience, *outcome, result.Changed, result.AlreadyResolved) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "email delivery reconciliation failed:", err) + os.Exit(1) +} diff --git a/cmd/email-delivery/main.go b/cmd/email-delivery/main.go new file mode 100644 index 0000000..c39dbfc --- /dev/null +++ b/cmd/email-delivery/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/emaildelivery" +) + +const ( + apiKeyEnvironment = "RESEND_API_KEY" + ledgerSecretEnvironment = "TRADEGRAVITY_DELIVERY_SECRET" +) + +func main() { + kitDir := flag.String("kit", "distribution-kit", "approved distribution-kit directory") + subscribers := flag.String("subscribers", "", "private double-opt-in subscriber CSV") + suppressions := flag.String("suppressions", "", "private suppression CSV") + preflight := flag.String("preflight", "delivery-preflight.json", "aggregate preflight bound to the launch authorization") + authorization := flag.String("authorization", "email-launch-authorization.json", "short-lived private launch authorization") + ledger := flag.String("ledger", "", "private SQLite delivery ledger") + sendAtValue := flag.String("send-at", "", "explicit RFC3339 send time inside the authorization window") + sendLive := flag.Bool("send-live", false, "make live provider requests; required in addition to all other gates") + flag.Parse() + + sendAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*sendAtValue)) + if err != nil { + fatal(fmt.Errorf("send-at must be RFC3339: %w", err)) + } + provider, err := emaildelivery.NewResendProvider(os.Getenv(apiKeyEnvironment), nil) + if err != nil { + fatal(err) + } + result, err := emaildelivery.Deliver(context.Background(), emaildelivery.DeliveryRequest{ + KitDir: *kitDir, + SubscriberCSV: *subscribers, + SuppressionCSV: *suppressions, + PreflightPath: *preflight, + AuthorizationPath: *authorization, + LedgerPath: *ledger, + LedgerSecret: []byte(os.Getenv(ledgerSecretEnvironment)), + SendAt: sendAt, + Provider: provider, + SendLive: *sendLive, + }) + if err != nil { + fatal(err) + } + fmt.Printf("email delivery complete (edition=%s audience=%s eligible=%d accepted=%d skipped=%d pending=%d)\n", result.EditionID, result.Audience, result.Eligible, result.Accepted, result.Skipped, result.Pending) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "email delivery failed:", err) + os.Exit(1) +} diff --git a/cmd/email-launch-approval/main.go b/cmd/email-launch-approval/main.go new file mode 100644 index 0000000..377bb9e --- /dev/null +++ b/cmd/email-launch-approval/main.go @@ -0,0 +1,63 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/emaildelivery" +) + +func main() { + preflight := flag.String("preflight", "delivery-preflight.json", "aggregate delivery preflight JSON") + output := flag.String("out", "email-launch-authorization.json", "new private launch authorization JSON") + provider := flag.String("provider", "resend", "email provider for this pilot authorization") + sender := flag.String("from", "", "authenticated sender mailbox") + replyTo := flag.String("reply-to", "", "optional reply-to mailbox") + authorizedBy := flag.String("authorized-by", "", "operator identity recorded in the launch authorization") + authorizedAtValue := flag.String("authorized-at", "", "explicit RFC3339 authorization time") + expiresAtValue := flag.String("expires-at", "", "explicit RFC3339 expiry no more than one hour later") + domainReady := flag.Bool("attest-domain-authenticated", false, "attest that SPF, DKIM, and DMARC were verified for the sender") + feedbackReady := flag.Bool("attest-feedback-ready", false, "attest that bounce and complaint handling is operational") + privacyReady := flag.Bool("attest-privacy-reviewed", false, "attest that retention, deletion, and operator access were reviewed") + pilotReady := flag.Bool("attest-pilot-recipients", false, "attest that every eligible pilot recipient was intentionally confirmed") + flag.Parse() + + authorizedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*authorizedAtValue)) + if err != nil { + fatal(fmt.Errorf("authorized-at must be RFC3339: %w", err)) + } + expiresAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*expiresAtValue)) + if err != nil { + fatal(fmt.Errorf("expires-at must be RFC3339: %w", err)) + } + authorization, raw, err := emaildelivery.Authorize(emaildelivery.AuthorizationRequest{ + PreflightPath: *preflight, + Provider: *provider, + Sender: *sender, + ReplyTo: *replyTo, + AuthorizedBy: *authorizedBy, + AuthorizedAt: authorizedAt, + ExpiresAt: expiresAt, + Attestations: emaildelivery.Attestations{ + SenderDomainAuthenticated: *domainReady, + BounceComplaintReady: *feedbackReady, + PrivacyControlsReviewed: *privacyReady, + PilotRecipientsConfirmed: *pilotReady, + }, + }) + if err != nil { + fatal(err) + } + if err := emaildelivery.WriteAuthorization(*output, raw); err != nil { + fatal(err) + } + fmt.Printf("email launch authorized (edition=%s audience=%s provider=%s eligible=%d expires_at=%s)\n", authorization.EditionID, authorization.Audience, authorization.Provider, authorization.EligibleRecipients, authorization.ExpiresAt) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "email launch approval failed:", err) + os.Exit(1) +} diff --git a/cmd/instagram-preflight/main.go b/cmd/instagram-preflight/main.go new file mode 100644 index 0000000..cf50906 --- /dev/null +++ b/cmd/instagram-preflight/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/socialpreflight" +) + +func main() { + kit := flag.String("kit", "distribution-kit", "Instagram-approved distribution-kit directory") + out := flag.String("out", "instagram-preflight.json", "new aggregate-only JSON outside the kit") + generated := flag.String("generated-at", "", "explicit RFC3339 preflight time") + flag.Parse() + generatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*generated)) + if err != nil { + fatal(fmt.Errorf("generated-at must be RFC3339: %w", err)) + } + result, err := socialpreflight.Build(*kit, generatedAt) + if err != nil { + fatal(err) + } + if err := socialpreflight.Write(*out, *kit, result.JSON); err != nil { + fatal(err) + } + fmt.Printf("Instagram preflight passed (edition=%s theme=%s slides=%d caption_runes=%d hashtags=%d manual_upload=%t automatic_publish=%t)\n", result.Plan.EditionID, result.Plan.Theme, result.Plan.SlideCount, result.Plan.CaptionRunes, result.Plan.HashtagCount, result.Plan.ManualUploadRequired, result.Plan.AutomaticPublishAuthorized) +} + +func fatal(err error) { fmt.Fprintln(os.Stderr, "Instagram preflight failed:", err); os.Exit(1) } diff --git a/cmd/publisher/briefing.go b/cmd/publisher/briefing.go new file mode 100644 index 0000000..2cc463e --- /dev/null +++ b/cmd/publisher/briefing.go @@ -0,0 +1,381 @@ +package main + +import ( + "fmt" + "math" + "sort" + "strings" + "time" +) + +const briefingSchemaVersion = "1.0" + +type briefingFile struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt string `json:"generated_at"` + EditionID string `json:"edition_id"` + Status string `json:"status"` + Title string `json:"title"` + Scope string `json:"scope"` + LatestPeriod string `json:"latest_period,omitempty"` + PreviousPeriod string `json:"previous_period,omitempty"` + PublicationStatus string `json:"publication_status"` + ReviewRequired bool `json:"review_required"` + Signals []briefingSignal `json:"signals"` + Email briefingEmail `json:"email"` + SocialCarousel briefingSocialCarousel `json:"social_carousel"` + Caveats []string `json:"caveats"` + EvidenceEntryPoint string `json:"evidence_entry_point"` +} + +type briefingSignal struct { + ID string `json:"id"` + Kind string `json:"kind"` + Title string `json:"title"` + Summary string `json:"summary"` + ReporterISO3 string `json:"reporter_iso3"` + ReporterName string `json:"reporter_name"` + Classification string `json:"classification,omitempty"` + Code string `json:"code,omitempty"` + Label string `json:"label,omitempty"` + Period string `json:"period"` + PreviousPeriod string `json:"previous_period"` + Current briefingObservedValue `json:"current"` + Previous briefingObservedValue `json:"previous"` + DeltaTradeUSD float64 `json:"delta_trade_usd"` + ChangeRatio *float64 `json:"change_ratio,omitempty"` + ChinaShareDelta float64 `json:"china_share_delta"` + Evidence []string `json:"evidence"` + Interpretation string `json:"interpretation"` + MeasurementLimit string `json:"measurement_limit"` +} + +type briefingObservedValue struct { + USATradeUSD float64 `json:"usa_trade_usd"` + ChinaTradeUSD float64 `json:"china_trade_usd"` + TotalTradeUSD float64 `json:"total_trade_usd"` + ChinaShare float64 `json:"china_share"` +} + +type briefingEmail struct { + Subject string `json:"subject"` + Preview string `json:"preview"` + Markdown string `json:"markdown"` + CTALabel string `json:"cta_label"` + CTAPath string `json:"cta_path"` + SendPolicy string `json:"send_policy"` + PrimaryGoal string `json:"primary_goal"` +} + +type briefingSocialCarousel struct { + Format string `json:"format"` + AspectRatio string `json:"aspect_ratio"` + ReviewPolicy string `json:"review_policy"` + Slides []briefingCarouselSlide `json:"slides"` +} + +type briefingCarouselSlide struct { + Order int `json:"order"` + Role string `json:"role"` + Headline string `json:"headline"` + Body string `json:"body"` + Evidence []string `json:"evidence"` +} + +type briefingCandidate struct { + signal briefingSignal + magnitude float64 +} + +func buildBriefing(generatedAt string, latest []latestEntry, monthlyIndex semiconductorMonthlyIndexFile, monthlyFiles map[string]semiconductorMonthlyFile, publicationChanges publicationChangesFile) briefingFile { + briefing := briefingFile{ + SchemaVersion: briefingSchemaVersion, + GeneratedAt: generatedAt, + EditionID: briefingEditionID(generatedAt, monthlyIndex.Periods), + Status: "unavailable", + Title: "TradeGravity Semiconductor Pulse", + Scope: "Deterministic distribution brief from selected monthly UN Comtrade HS6 observations against USA and China; not a complete semiconductor market, causal claim, or investment recommendation", + PublicationStatus: publicationChanges.Status, + ReviewRequired: true, + Signals: []briefingSignal{}, + EvidenceEntryPoint: "./?tab=semiconductors", + Caveats: []string{ + "Monthly customs observations can be volatile, incomplete, and revised.", + "USA and China values are the two published anchor relationships, not world totals or physical shipment routes.", + "Publication-to-publication revisions are separate from economic month-to-month movement.", + }, + Email: briefingEmail{ + CTALabel: "Inspect the evidence", + CTAPath: "./?tab=semiconductors", + SendPolicy: "manual_review_required", + PrimaryGoal: "Return the reader to the cited TradeGravity evidence", + }, + SocialCarousel: briefingSocialCarousel{ + Format: "carousel_copy", + AspectRatio: "4:5", + ReviewPolicy: "manual_review_required", + Slides: []briefingCarouselSlide{}, + }, + } + if len(monthlyIndex.Periods) > 0 { + briefing.LatestPeriod = monthlyIndex.Periods[len(monthlyIndex.Periods)-1] + } + if len(monthlyIndex.Periods) > 1 { + briefing.PreviousPeriod = monthlyIndex.Periods[len(monthlyIndex.Periods)-2] + } + + names := make(map[string]string, len(latest)) + for _, row := range latest { + name := strings.TrimSpace(row.Name) + if name == "" { + name = row.ISO3 + } + names[strings.ToUpper(row.ISO3)] = name + } + reporterCandidates := make([]briefingCandidate, 0, len(monthlyFiles)) + shareCandidates := make([]briefingCandidate, 0, len(monthlyFiles)) + productCandidates := make([]briefingCandidate, 0) + currentPeriod, previousPeriod := briefing.LatestPeriod, briefing.PreviousPeriod + if currentPeriod == "" || previousPeriod == "" { + briefing.Email.Subject = "TradeGravity Semiconductor Pulse · data unavailable" + briefing.Email.Preview = "The current publication does not contain enough comparable monthly observations to produce a distribution brief." + briefing.Email.Markdown = "# TradeGravity Semiconductor Pulse\n\nNo distribution brief was generated because two comparable monthly observations were not available. This is not interpreted as no change.\n" + return briefing + } + for _, file := range monthlyFiles { + currentAggregate, currentOK := aggregateBriefingRows(file.Rows, currentPeriod) + previousAggregate, previousOK := aggregateBriefingRows(file.Rows, previousPeriod) + if !currentOK || !previousOK { + continue + } + reporter := strings.ToUpper(strings.TrimSpace(file.ReporterISO3)) + name := names[reporter] + if name == "" { + name = reporter + } + reporterSignal := makeBriefingSignal("reporter_total_change", reporter, name, "", "", "", currentPeriod, previousPeriod, currentAggregate, previousAggregate) + reporterSignal.ID = "monthly-total-" + strings.ToLower(reporter) + reporterSignal.Title = fmt.Sprintf("%s selected chip trade %s", name, movementWord(reporterSignal.DeltaTradeUSD)) + reporterSignal.Summary = fmt.Sprintf("Selected monthly HS6 trade with USA and China moved from %s to %s (%s).", formatBriefingUSD(previousAggregate.TotalTradeUSD), formatBriefingUSD(currentAggregate.TotalTradeUSD), formatBriefingPercent(reporterSignal.ChangeRatio)) + reporterSignal.Interpretation = "A change in the selected two-anchor customs observations worth investigating; it does not establish production, demand, or causality." + reporterSignal.Evidence = []string{"./semiconductors/monthly/" + reporter + ".json", "./semiconductors/monthly/index.json"} + reporterCandidates = append(reporterCandidates, briefingCandidate{signal: reporterSignal, magnitude: math.Abs(reporterSignal.DeltaTradeUSD)}) + + shareSignal := reporterSignal + shareSignal.ID = "anchor-share-" + strings.ToLower(reporter) + shareSignal.Kind = "anchor_share_shift" + shareSignal.Title = fmt.Sprintf("%s two-anchor balance shifted %s", name, anchorDirection(shareSignal.ChinaShareDelta)) + shareSignal.Summary = fmt.Sprintf("China's share of the selected USA-plus-China total moved from %.1f%% to %.1f%% (%+.1f percentage points).", previousAggregate.ChinaShare*100, currentAggregate.ChinaShare*100, shareSignal.ChinaShareDelta*100) + shareSignal.Interpretation = "The sign describes movement within the published USA-China anchor pair, not political alignment or global market share." + shareCandidates = append(shareCandidates, briefingCandidate{signal: shareSignal, magnitude: math.Abs(shareSignal.ChinaShareDelta)}) + + currentProducts := briefingProductsByKey(file.Rows, currentPeriod) + previousProducts := briefingProductsByKey(file.Rows, previousPeriod) + for key, current := range currentProducts { + previous, ok := previousProducts[key] + if !ok { + continue + } + productSignal := makeBriefingSignal("product_total_change", reporter, name, current.Classification, current.Code, current.Label, currentPeriod, previousPeriod, observedValueFromMonthly(current), observedValueFromMonthly(previous)) + productSignal.ID = "product-" + strings.ToLower(reporter) + "-" + current.Code + productSignal.Title = fmt.Sprintf("%s · %s %s", name, current.Label, movementWord(productSignal.DeltaTradeUSD)) + productSignal.Summary = fmt.Sprintf("HS6 %s selected trade moved from %s to %s (%s).", current.Code, formatBriefingUSD(productSignal.Previous.TotalTradeUSD), formatBriefingUSD(productSignal.Current.TotalTradeUSD), formatBriefingPercent(productSignal.ChangeRatio)) + productSignal.Interpretation = "This is a product-level customs observation against USA and China, not company revenue, capacity, or a shipment route." + productSignal.Evidence = []string{"./semiconductors/monthly/" + reporter + ".json", "./semiconductors/reference.json"} + productCandidates = append(productCandidates, briefingCandidate{signal: productSignal, magnitude: math.Abs(productSignal.DeltaTradeUSD)}) + } + } + + sortBriefingCandidates(reporterCandidates) + sortBriefingCandidates(shareCandidates) + sortBriefingCandidates(productCandidates) + for _, candidates := range [][]briefingCandidate{reporterCandidates, shareCandidates, productCandidates} { + if len(candidates) > 0 { + briefing.Signals = append(briefing.Signals, candidates[0].signal) + } + } + if len(briefing.Signals) != 3 { + briefing.Email.Subject = "TradeGravity Semiconductor Pulse · data unavailable" + briefing.Email.Preview = "The current publication does not contain enough comparable monthly observations to produce a distribution brief." + briefing.Email.Markdown = "# TradeGravity Semiconductor Pulse\n\nNo distribution brief was generated because two comparable monthly observations were not available. This is not interpreted as no change.\n" + return briefing + } + + briefing.Status = "ready" + briefing.Email = buildBriefingEmail(briefing) + briefing.SocialCarousel = buildBriefingCarousel(briefing) + return briefing +} + +func aggregateBriefingRows(rows []semiconductorMonthlyProductEntry, period string) (briefingObservedValue, bool) { + value := briefingObservedValue{} + found := false + for _, row := range rows { + if row.Period != period { + continue + } + value.USATradeUSD += row.USA.Trade + value.ChinaTradeUSD += row.CHN.Trade + found = true + } + value.TotalTradeUSD = value.USATradeUSD + value.ChinaTradeUSD + if value.TotalTradeUSD > 0 { + value.ChinaShare = value.ChinaTradeUSD / value.TotalTradeUSD + } + return value, found +} + +func briefingProductsByKey(rows []semiconductorMonthlyProductEntry, period string) map[string]semiconductorMonthlyProductEntry { + result := make(map[string]semiconductorMonthlyProductEntry) + for _, row := range rows { + if row.Period != period { + continue + } + key := strings.ToUpper(strings.TrimSpace(row.Classification)) + "|" + row.Code + result[key] = row + } + return result +} + +func observedValueFromMonthly(row semiconductorMonthlyProductEntry) briefingObservedValue { + return briefingObservedValue{USATradeUSD: row.USA.Trade, ChinaTradeUSD: row.CHN.Trade, TotalTradeUSD: row.Total, ChinaShare: row.ShareCN} +} + +func makeBriefingSignal(kind, reporter, name, classification, code, label, period, previousPeriod string, current, previous briefingObservedValue) briefingSignal { + delta := current.TotalTradeUSD - previous.TotalTradeUSD + var ratio *float64 + if previous.TotalTradeUSD > 0 { + value := delta / previous.TotalTradeUSD + ratio = &value + } + return briefingSignal{ + Kind: kind, + ReporterISO3: reporter, + ReporterName: name, + Classification: classification, + Code: code, + Label: label, + Period: period, + PreviousPeriod: previousPeriod, + Current: current, + Previous: previous, + DeltaTradeUSD: delta, + ChangeRatio: ratio, + ChinaShareDelta: current.ChinaShare - previous.ChinaShare, + Evidence: []string{}, + MeasurementLimit: "Selected monthly HS6 observations against USA and China only; subject to source revisions and coverage limits.", + } +} + +func sortBriefingCandidates(candidates []briefingCandidate) { + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].magnitude != candidates[j].magnitude { + return candidates[i].magnitude > candidates[j].magnitude + } + left, right := candidates[i].signal, candidates[j].signal + if left.ReporterISO3 != right.ReporterISO3 { + return left.ReporterISO3 < right.ReporterISO3 + } + return left.Code < right.Code + }) +} + +func buildBriefingEmail(briefing briefingFile) briefingEmail { + lines := []string{ + "# " + briefing.Title, + "", + fmt.Sprintf("Observation window: %s vs %s · publication status: %s", briefing.LatestPeriod, briefing.PreviousPeriod, briefing.PublicationStatus), + "", + } + for _, signal := range briefing.Signals { + lines = append(lines, "## "+signal.Title, "", signal.Summary, "", "Interpretation boundary: "+signal.Interpretation, "") + } + lines = append(lines, + "Review the cited evidence before forwarding or publishing this draft.", + "", + "[Inspect the evidence]({{BASE_URL}}/?tab=semiconductors)", + "", + "Data scope: selected monthly UN Comtrade HS6 observations against USA and China. Not investment, legal, or policy advice.", + ) + return briefingEmail{ + Subject: fmt.Sprintf("TradeGravity Semiconductor Pulse · %s", briefing.LatestPeriod), + Preview: fmt.Sprintf("Three cited USA-China semiconductor observations for %s; monthly movement is kept separate from publication revisions.", briefing.LatestPeriod), + Markdown: strings.Join(lines, "\n"), + CTALabel: "Inspect the evidence", + CTAPath: "./?tab=semiconductors", + SendPolicy: "manual_review_required", + PrimaryGoal: "Return the reader to the cited TradeGravity evidence", + } +} + +func buildBriefingCarousel(briefing briefingFile) briefingSocialCarousel { + slides := []briefingCarouselSlide{{ + Order: 1, Role: "cover", Headline: briefing.Title, + Body: fmt.Sprintf("Three USA-China semiconductor observations · %s vs %s", briefing.LatestPeriod, briefing.PreviousPeriod), + Evidence: []string{"./semiconductors/monthly/index.json"}, + }} + roles := []string{"scale", "anchor_balance", "product"} + for index, signal := range briefing.Signals { + slides = append(slides, briefingCarouselSlide{Order: index + 2, Role: roles[index], Headline: signal.Title, Body: signal.Summary, Evidence: append([]string(nil), signal.Evidence...)}) + } + slides = append(slides, + briefingCarouselSlide{Order: 5, Role: "method", Headline: "Read the clocks separately", Body: "Month-to-month customs movement and publish-to-publish revisions answer different questions. Neither proves causality or a physical route.", Evidence: []string{"./changes.json", "./semiconductors/monthly/index.json"}}, + briefingCarouselSlide{Order: 6, Role: "cta", Headline: "Inspect the evidence", Body: "Open TradeGravity's Chip Lens for periods, values, sources, coverage, and limitations.", Evidence: []string{"./?tab=semiconductors"}}, + ) + return briefingSocialCarousel{Format: "carousel_copy", AspectRatio: "4:5", ReviewPolicy: "manual_review_required", Slides: slides} +} + +func briefingEditionID(generatedAt string, periods []string) string { + period := "no-period" + if len(periods) > 0 { + period = periods[len(periods)-1] + } + timestamp := "unknown" + if parsed, err := time.Parse(time.RFC3339, generatedAt); err == nil { + timestamp = parsed.UTC().Format("20060102T150405Z") + } + return "semiconductor-pulse-" + period + "-" + timestamp +} + +func movementWord(delta float64) string { + if delta > 0 { + return "increased" + } + if delta < 0 { + return "decreased" + } + return "was unchanged" +} + +func anchorDirection(chinaShareDelta float64) string { + if chinaShareDelta > 0 { + return "toward China" + } + if chinaShareDelta < 0 { + return "toward USA" + } + return "without a change" +} + +func formatBriefingUSD(value float64) string { + abs := math.Abs(value) + switch { + case abs >= 1e12: + return fmt.Sprintf("US$%.2fT", value/1e12) + case abs >= 1e9: + return fmt.Sprintf("US$%.2fB", value/1e9) + case abs >= 1e6: + return fmt.Sprintf("US$%.2fM", value/1e6) + case abs >= 1e3: + return fmt.Sprintf("US$%.2fK", value/1e3) + default: + return fmt.Sprintf("US$%.0f", value) + } +} + +func formatBriefingPercent(value *float64) string { + if value == nil { + return "no comparable percentage" + } + return fmt.Sprintf("%+.1f%%", *value*100) +} diff --git a/cmd/publisher/briefing_test.go b/cmd/publisher/briefing_test.go new file mode 100644 index 0000000..d8a1b9c --- /dev/null +++ b/cmd/publisher/briefing_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "strings" + "testing" +) + +func TestBuildBriefingCreatesDeterministicEmailAndCarouselDrafts(t *testing.T) { + generatedAt := "2026-07-17T00:00:00Z" + index := semiconductorMonthlyIndexFile{Periods: []string{"2026-04", "2026-05"}, Reporters: []string{"JPN", "KOR"}} + files := map[string]semiconductorMonthlyFile{ + "KOR.json": briefingMonthlyFile("KOR", []semiconductorMonthlyProductEntry{ + briefingMonthlyRow("2026-04", "854232", "Memories", 100, 100), + briefingMonthlyRow("2026-04", "848620", "Semiconductor manufacturing machinery", 50, 50), + briefingMonthlyRow("2026-05", "854232", "Memories", 180, 120), + briefingMonthlyRow("2026-05", "848620", "Semiconductor manufacturing machinery", 55, 45), + }), + "JPN.json": briefingMonthlyFile("JPN", []semiconductorMonthlyProductEntry{ + briefingMonthlyRow("2026-04", "854232", "Memories", 80, 120), + briefingMonthlyRow("2026-05", "854232", "Memories", 120, 80), + }), + } + latest := []latestEntry{{ISO3: "KOR", Name: "Korea, Rep."}, {ISO3: "JPN", Name: "Japan"}} + changes := publicationChangesFile{Status: "changed"} + + got := buildBriefing(generatedAt, latest, index, files, changes) + if got.Status != "ready" || got.SchemaVersion != "1.0" || got.EditionID != "semiconductor-pulse-2026-05-20260717T000000Z" { + t.Fatalf("unexpected briefing identity: %+v", got) + } + if !got.ReviewRequired || got.PublicationStatus != "changed" || got.LatestPeriod != "2026-05" || got.PreviousPeriod != "2026-04" { + t.Fatalf("unexpected briefing provenance: %+v", got) + } + if len(got.Signals) != 3 { + t.Fatalf("signals = %d, want 3", len(got.Signals)) + } + if got.Signals[0].Kind != "reporter_total_change" || got.Signals[0].ReporterISO3 != "KOR" || got.Signals[0].DeltaTradeUSD != 100 { + t.Fatalf("unexpected scale signal: %+v", got.Signals[0]) + } + if got.Signals[1].Kind != "anchor_share_shift" || got.Signals[1].ReporterISO3 != "JPN" { + t.Fatalf("unexpected anchor signal: %+v", got.Signals[1]) + } + assertFloat(t, "anchor share shift", got.Signals[1].ChinaShareDelta, -0.2) + if got.Signals[2].Kind != "product_total_change" || got.Signals[2].ReporterISO3 != "KOR" || got.Signals[2].Code != "854232" || got.Signals[2].DeltaTradeUSD != 100 { + t.Fatalf("unexpected product signal: %+v", got.Signals[2]) + } + if got.Email.SendPolicy != "manual_review_required" || !strings.Contains(got.Email.Markdown, "{{BASE_URL}}") || !strings.Contains(got.Email.Markdown, got.Signals[0].Title) { + t.Fatalf("email draft does not preserve review and evidence handoff: %+v", got.Email) + } + if got.SocialCarousel.AspectRatio != "4:5" || got.SocialCarousel.ReviewPolicy != "manual_review_required" || len(got.SocialCarousel.Slides) != 6 { + t.Fatalf("unexpected carousel draft: %+v", got.SocialCarousel) + } + for index, slide := range got.SocialCarousel.Slides { + if slide.Order != index+1 || len(slide.Evidence) == 0 { + t.Fatalf("slide %d is not ordered or cited: %+v", index, slide) + } + } +} + +func TestBuildBriefingFailsClosedWithoutTwoComparableMonths(t *testing.T) { + index := semiconductorMonthlyIndexFile{Periods: []string{"2026-05"}, Reporters: []string{"KOR"}} + files := map[string]semiconductorMonthlyFile{ + "KOR.json": briefingMonthlyFile("KOR", []semiconductorMonthlyProductEntry{briefingMonthlyRow("2026-05", "854232", "Memories", 100, 100)}), + } + got := buildBriefing("2026-07-17T00:00:00Z", []latestEntry{{ISO3: "KOR", Name: "Korea, Rep."}}, index, files, publicationChangesFile{Status: "baseline"}) + if got.Status != "unavailable" || len(got.Signals) != 0 || len(got.SocialCarousel.Slides) != 0 { + t.Fatalf("briefing should fail closed: %+v", got) + } + if !strings.Contains(got.Email.Markdown, "not available") || got.Email.SendPolicy != "manual_review_required" { + t.Fatalf("unavailable email state is not explicit: %+v", got.Email) + } +} + +func TestBuildBriefingUsesOnePublicationWindowAcrossReporters(t *testing.T) { + index := semiconductorMonthlyIndexFile{Periods: []string{"2026-03", "2026-04", "2026-05"}, Reporters: []string{"JPN", "KOR"}} + files := map[string]semiconductorMonthlyFile{ + "JPN.json": briefingMonthlyFile("JPN", []semiconductorMonthlyProductEntry{ + briefingMonthlyRow("2026-04", "854232", "Memories", 100, 100), + briefingMonthlyRow("2026-05", "854232", "Memories", 120, 80), + }), + "KOR.json": { + ReporterISO3: "KOR", + Periods: []string{"2026-03", "2026-04"}, + Rows: []semiconductorMonthlyProductEntry{ + briefingMonthlyRow("2026-03", "854232", "Memories", 100, 100), + briefingMonthlyRow("2026-04", "854232", "Memories", 1000, 0), + }, + }, + } + latest := []latestEntry{{ISO3: "JPN", Name: "Japan"}, {ISO3: "KOR", Name: "Korea, Rep."}} + + got := buildBriefing("2026-07-17T00:00:00Z", latest, index, files, publicationChangesFile{Status: "changed"}) + if got.Status != "ready" || got.LatestPeriod != "2026-05" || got.PreviousPeriod != "2026-04" { + t.Fatalf("unexpected common publication window: %+v", got) + } + for _, signal := range got.Signals { + if signal.ReporterISO3 != "JPN" || signal.Period != "2026-05" || signal.PreviousPeriod != "2026-04" { + t.Fatalf("signal escaped the common publication window: %+v", signal) + } + } +} + +func briefingMonthlyFile(reporter string, rows []semiconductorMonthlyProductEntry) semiconductorMonthlyFile { + return semiconductorMonthlyFile{ReporterISO3: reporter, Periods: []string{"2026-04", "2026-05"}, Rows: rows} +} + +func briefingMonthlyRow(period, code, label string, usa, china float64) semiconductorMonthlyProductEntry { + total := usa + china + share := 0.0 + if total > 0 { + share = china / total + } + return semiconductorMonthlyProductEntry{ + Period: period, Classification: "H6", Code: code, Label: label, + USA: seriesBlock{Available: true, Trade: usa}, CHN: seriesBlock{Available: true, Trade: china}, Total: total, ShareCN: share, + } +} diff --git a/cmd/publisher/extended.go b/cmd/publisher/extended.go index 7fc07de..df776ab 100644 --- a/cmd/publisher/extended.go +++ b/cmd/publisher/extended.go @@ -347,7 +347,7 @@ type providerComparison struct { DeltaRatio float64 `json:"delta_ratio"` } -func buildDataCatalog(generatedAt, provider, contextStatus string, series seriesFile, products productIndexFile, strategicIndex strategicIndexFile, tariffIndex tariffIndexFile, matrixIndex matrixIndexFile, mirrorIndex mirrorIndexFile, semiconductorMonthlyIndex semiconductorMonthlyIndexFile, publicationChanges publicationChangesFile, semiconductorReferences ...semiconductor.Reference) dataCatalogFile { +func buildDataCatalog(generatedAt, provider, contextStatus string, series seriesFile, products productIndexFile, strategicIndex strategicIndexFile, tariffIndex tariffIndexFile, matrixIndex matrixIndexFile, mirrorIndex mirrorIndexFile, semiconductorMonthlyIndex semiconductorMonthlyIndexFile, publicationChanges publicationChangesFile, briefing briefingFile, semiconductorReferences ...semiconductor.Reference) dataCatalogFile { semiconductorReference := semiconductor.Reference{} if len(semiconductorReferences) > 0 { semiconductorReference = semiconductorReferences[0] @@ -389,6 +389,10 @@ func buildDataCatalog(generatedAt, provider, contextStatus string, series series if publicationChanges.Status == "changed" || publicationChanges.Status == "unchanged" { publicationChangesStatus = "ready" } + briefingStatus := "partial" + if briefing.Status == "ready" { + briefingStatus = "ready" + } return dataCatalogFile{ SchemaVersion: "1.0", GeneratedAt: generatedAt, @@ -404,6 +408,7 @@ func buildDataCatalog(generatedAt, provider, contextStatus string, series series {ID: "semiconductor_atlas", Title: "Semiconductor value-chain atlas", Status: semiconductorStatus, Provider: "tradegravity + cited official sources", Classification: "stage-mapped source HS revision", ProductLevel: 6, Grain: "stage taxonomy + country role context + policy event + published HS6 coverage", Partitioning: "reference publication + strategic HS6 reporter/year chunks", Href: "./semiconductors/reference.json"}, {ID: "semiconductor_monthly", Title: "Focused US-China semiconductor turning points", Status: semiconductorMonthlyStatus, Provider: semiconductorMonthlyIndex.Provider, Classification: "source HS revision", ProductLevel: 6, Grain: "focused reporter × USA/CHN partner × flow × selected HS6 × month", Partitioning: "index + one file per reporter", Href: "./semiconductors/monthly/index.json"}, {ID: "publication_changes", Title: "Observed publication changes", Status: publicationChangesStatus, Provider: "tradegravity", Classification: "source HS revision", ProductLevel: 6, Grain: "publication × focused reporter × month × selected HS6", Partitioning: "single bounded change feed", Href: "./changes.json"}, + {ID: "distribution_briefing", Title: "Deterministic email and social briefing", Status: briefingStatus, Provider: "tradegravity", Classification: "source HS revision", ProductLevel: 6, Grain: "edition × selected monthly observation × distribution channel", Partitioning: "single reviewed-draft publication", Href: "./briefing.json"}, {ID: "mirror_reconciliation", Title: "Unadjusted mirror-reporting diagnostics", Status: mirrorStatus, Provider: mirrorIndex.Provider, ProductLevel: 0, Grain: "third-country reporter × USA/CHN anchor × mirrored flow × TOTAL × annual period", Partitioning: "index + reporter/year chunks", Href: "./mirror/index.json"}, {ID: "value_added_network", Title: "Value-added supply-chain exposure", Status: "planned", Grain: "origin × destination × industry × year", Partitioning: "year/industry chunks"}, {ID: "scenario_runs", Title: "Versioned scenario outputs", Status: "planned", Grain: "scenario × market × product × partner", Partitioning: "one manifest and result set per run"}, diff --git a/cmd/publisher/extended_test.go b/cmd/publisher/extended_test.go index f4f6f0a..30f880c 100644 --- a/cmd/publisher/extended_test.go +++ b/cmd/publisher/extended_test.go @@ -78,6 +78,7 @@ func TestBuildDataCatalogSeparatesReadyAndPlannedResources(t *testing.T) { mirrorIndexFile{Provider: "comtrade", Partitions: []mirrorPartition{{ReporterISO3: "KOR", Period: "2023"}}}, semiconductorMonthlyIndexFile{Provider: "comtrade", Partitions: []semiconductorMonthlyPartition{{ReporterISO3: "KOR"}}}, publicationChangesFile{Status: "changed"}, + briefingFile{Status: "ready"}, ) if catalog.SchemaVersion != "1.0" || len(catalog.Resources) < 10 { t.Fatalf("unexpected catalog shape: %+v", catalog) @@ -104,6 +105,9 @@ func TestBuildDataCatalogSeparatesReadyAndPlannedResources(t *testing.T) { if byID["publication_changes"].Status != "ready" || byID["publication_changes"].Href != "./changes.json" { t.Fatalf("publication change feed is not published: %+v", byID["publication_changes"]) } + if byID["distribution_briefing"].Status != "ready" || byID["distribution_briefing"].Href != "./briefing.json" { + t.Fatalf("distribution briefing is not published: %+v", byID["distribution_briefing"]) + } } func TestBuildMirrorFilesComparesBothReportedDirectionsWithoutChoosingTruth(t *testing.T) { diff --git a/cmd/publisher/main.go b/cmd/publisher/main.go index 985566f..eae0062 100644 --- a/cmd/publisher/main.go +++ b/cmd/publisher/main.go @@ -234,6 +234,7 @@ func build(args []string) { fmt.Fprintln(os.Stderr, "failed to compare the previous semiconductor publication:", err) os.Exit(1) } + briefing := buildBriefing(now, latest, semiconductorMonthlyIndex, semiconductorMonthlyFiles, publicationChanges) tariffRows, err := loadTariffObservations(*dbPath, "trains") if err != nil { fmt.Fprintln(os.Stderr, "failed to load tariff observations:", err) @@ -253,7 +254,7 @@ func build(args []string) { os.Exit(1) } quality := buildQualityFile(now, *provider, latest, rows, productRows, runs) - catalog := buildDataCatalog(now, *provider, contextData.Status, seriesOutput, productIndex, strategicIndex, tariffIndex, matrixIndex, mirrorIndex, semiconductorMonthlyIndex, publicationChanges, semiconductorReference) + catalog := buildDataCatalog(now, *provider, contextData.Status, seriesOutput, productIndex, strategicIndex, tariffIndex, matrixIndex, mirrorIndex, semiconductorMonthlyIndex, publicationChanges, briefing, semiconductorReference) metadata := buildMeta(now, *provider, partners, rows, latest) augmentMeta(&metadata, latest, seriesOutput, productIndex, len(productRows), contextData.Status) augmentStrategicMeta(&metadata, strategicIndex) @@ -294,6 +295,10 @@ func build(args []string) { fmt.Fprintln(os.Stderr, "failed to write changes.json:", err) os.Exit(1) } + if err := writeJSON(filepath.Join(*outDir, "briefing.json"), briefing); err != nil { + fmt.Fprintln(os.Stderr, "failed to write briefing.json:", err) + os.Exit(1) + } productsDir := filepath.Join(*outDir, "products") if err := os.MkdirAll(productsDir, 0o755); err != nil { fmt.Fprintln(os.Stderr, "failed to create products dir:", err) diff --git a/cmd/subscription-registry/main.go b/cmd/subscription-registry/main.go new file mode 100644 index 0000000..425f079 --- /dev/null +++ b/cmd/subscription-registry/main.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "strings" + "time" + + "tradegravity/internal/subscriptions" +) + +const secretEnvironment = "TRADEGRAVITY_UNSUBSCRIBE_SECRET" + +func main() { + databasePath := flag.String("db", "", "private SQLite subscription database path") + publicBaseURL := flag.String("base-url", "", "public HTTPS base URL for the unsubscribe service") + consentPath := flag.String("consents", "", "private active double-opt-in consent CSV") + audience := flag.String("audience", "", "non-sensitive audience label to export") + subscriberOutput := flag.String("out-subscribers", "", "new private delivery subscriber CSV") + suppressionOutput := flag.String("out-suppressions", "", "new private suppression CSV") + importedAtValue := flag.String("imported-at", "", "explicit RFC3339 import time") + flag.Parse() + + importedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(*importedAtValue)) + if err != nil { + fatal(fmt.Errorf("imported-at must be RFC3339: %w", err)) + } + secret := []byte(os.Getenv(secretEnvironment)) + if len(secret) == 0 { + fatal(fmt.Errorf("%s is required", secretEnvironment)) + } + consents, err := readPrivateInput(*consentPath) + if err != nil { + fatal(err) + } + registry, err := subscriptions.Open(*databasePath, secret, *publicBaseURL) + if err != nil { + fatal(err) + } + defer registry.Close() + + result, err := registry.ImportConsents(context.Background(), consents, importedAt) + if err != nil { + fatal(err) + } + subscribersCSV, suppressionsCSV, err := registry.ExportAudience(context.Background(), *audience) + if err != nil { + fatal(err) + } + if err := subscriptions.WritePrivateExports(*subscriberOutput, subscribersCSV, *suppressionOutput, suppressionsCSV); err != nil { + fatal(err) + } + fmt.Printf("subscription registry updated (inserted=%d updated=%d suppressed_skipped=%d exports=2)\n", result.Inserted, result.Updated, result.SuppressedSkipped) +} + +func readPrivateInput(path string) ([]byte, error) { + path = strings.TrimSpace(path) + if path == "" { + return nil, errors.New("consent CSV path is required") + } + info, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("inspect consent CSV: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return nil, errors.New("consent CSV must be a regular non-symlink file") + } + if info.Size() > 5<<20 { + return nil, errors.New("consent CSV exceeds the 5 MiB limit") + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read consent CSV: %w", err) + } + return raw, nil +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "subscription registry failed:", err) + os.Exit(1) +} diff --git a/cmd/unsubscribe-service/main.go b/cmd/unsubscribe-service/main.go new file mode 100644 index 0000000..95c509b --- /dev/null +++ b/cmd/unsubscribe-service/main.go @@ -0,0 +1,145 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "html" + "log" + "net/http" + "net/mail" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "tradegravity/internal/emaildelivery" + "tradegravity/internal/subscriptions" +) + +const ( + secretEnvironment = "TRADEGRAVITY_UNSUBSCRIBE_SECRET" + webhookSecretEnvironment = "RESEND_WEBHOOK_SECRET" + resendAPIKeyEnvironment = "RESEND_API_KEY" +) + +type resendConfirmationSender struct { + provider *emaildelivery.ResendProvider + from, replyTo string +} + +func (sender resendConfirmationSender) SendConfirmation(ctx context.Context, confirmation subscriptions.ConfirmationEmail) (string, error) { + safeURL := html.EscapeString(confirmation.ConfirmationURL) + message := emaildelivery.Message{ + From: sender.from, ReplyTo: sender.replyTo, To: confirmation.To, + Subject: "Confirm your TradeGravity briefing subscription", + HTML: `
You requested the TradeGravity research briefing.
Review and confirm your subscription
Opening the link does not subscribe you; use the confirmation button on that page. This link expires soon. If you did not request it, ignore this email.
`, + Text: "You requested the TradeGravity research briefing.\n\nReview and confirm your subscription:\n" + confirmation.ConfirmationURL + "\n\nOpening the link does not subscribe you; use the confirmation button on that page. This link expires soon. If you did not request it, ignore this email.\n", + } + return sender.provider.Send(ctx, message, confirmation.IdempotencyKey) +} + +func main() { + databasePath := flag.String("db", "", "private SQLite subscription database path") + publicBaseURL := flag.String("base-url", "", "public HTTPS base URL for links generated by the registry") + listenAddress := flag.String("listen", "127.0.0.1:8081", "HTTP listen address; use a TLS reverse proxy for public service") + enableResendWebhook := flag.Bool("enable-resend-webhook", false, "enable the signed Resend feedback endpoint") + enableSignup := flag.Bool("enable-signup", false, "enable the public double-opt-in subscription form") + signupAudience := flag.String("signup-audience", "tradegravity-briefing", "fixed audience for the public form") + consentSource := flag.String("consent-source", "public-subscribe-form", "consent evidence source label") + privacyVersion := flag.String("privacy-notice-version", "v1", "privacy notice version recorded with consent") + privacyURL := flag.String("privacy-notice-url", "", "public HTTPS privacy notice URL") + confirmationFrom := flag.String("confirmation-from", "", "verified sender used for confirmation mail") + confirmationReplyTo := flag.String("confirmation-reply-to", "", "optional reply-to address") + confirmationTTL := flag.Duration("confirmation-ttl", 30*time.Minute, "confirmation link lifetime (5m to 24h)") + maximumPending := flag.Int("max-pending", 100, "maximum unexpired pending confirmations") + flag.Parse() + + secret := []byte(os.Getenv(secretEnvironment)) + if len(secret) == 0 { + fatal(fmt.Errorf("%s is required", secretEnvironment)) + } + registry, err := subscriptions.Open(*databasePath, secret, *publicBaseURL) + if err != nil { + fatal(err) + } + defer registry.Close() + + options := subscriptions.HandlerOptions{Now: time.Now} + if *enableResendWebhook { + webhookSecret := os.Getenv(webhookSecretEnvironment) + if strings.TrimSpace(webhookSecret) == "" { + fatal(fmt.Errorf("%s is required when the Resend webhook is enabled", webhookSecretEnvironment)) + } + options.ResendWebhookSecret = webhookSecret + } + if *enableSignup { + provider, providerErr := emaildelivery.NewResendProvider(os.Getenv(resendAPIKeyEnvironment), nil) + if providerErr != nil { + fatal(providerErr) + } + if strings.TrimSpace(*confirmationFrom) == "" { + fatal(errors.New("confirmation-from is required when signup is enabled")) + } + if _, parseErr := mail.ParseAddress(*confirmationFrom); parseErr != nil { + fatal(errors.New("confirmation-from is invalid")) + } + if strings.TrimSpace(*confirmationReplyTo) != "" { + if _, parseErr := mail.ParseAddress(*confirmationReplyTo); parseErr != nil { + fatal(errors.New("confirmation-reply-to is invalid")) + } + } + options.Signup = &subscriptions.SignupOptions{Config: subscriptions.SignupConfig{ + Audience: *signupAudience, ConsentSource: *consentSource, PrivacyNoticeVersion: *privacyVersion, + PrivacyNoticeURL: *privacyURL, ConfirmationTTL: *confirmationTTL, DispatchCooldown: time.Minute, MaxPending: *maximumPending, + }, Sender: resendConfirmationSender{provider: provider, from: *confirmationFrom, replyTo: *confirmationReplyTo}} + } + handler, err := registry.HandlerWithOptions(options) + if err != nil { + fatal(err) + } + server := &http.Server{ + Addr: strings.TrimSpace(*listenAddress), + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 30 * time.Second, + MaxHeaderBytes: 8 << 10, + ErrorLog: log.New(os.Stderr, "unsubscribe service: ", 0), + } + if server.Addr == "" { + fatal(errors.New("listen address is required")) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + done := make(chan error, 1) + go func() { + done <- server.ListenAndServe() + }() + fmt.Printf("subscription service listening (address=%s tls=reverse-proxy-required resend_webhook=%t signup=%t)\n", server.Addr, *enableResendWebhook, *enableSignup) + + select { + case err := <-done: + if !errors.Is(err, http.ErrServerClosed) { + fatal(err) + } + case <-ctx.Done(): + shutdownContext, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := server.Shutdown(shutdownContext); err != nil { + fatal(err) + } + if err := <-done; !errors.Is(err, http.ErrServerClosed) { + fatal(err) + } + } +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "unsubscribe service failed:", err) + os.Exit(1) +} diff --git a/cmd/validator/briefing_test.go b/cmd/validator/briefing_test.go new file mode 100644 index 0000000..56fbbb3 --- /dev/null +++ b/cmd/validator/briefing_test.go @@ -0,0 +1,49 @@ +package main + +import "testing" + +func TestValidateBriefingAcceptsReviewGatedCitedDraft(t *testing.T) { + metadata := datasetMeta{GeneratedAt: "2026-07-17T00:00:00Z"} + briefing := validValidationBriefing(metadata.GeneratedAt) + if err := validateBriefing(metadata, validationPublicationChanges{Status: "changed"}, briefing); err != nil { + t.Fatalf("validateBriefing() error = %v", err) + } +} + +func TestValidateBriefingRejectsAutomaticSendAndBrokenArithmetic(t *testing.T) { + metadata := datasetMeta{GeneratedAt: "2026-07-17T00:00:00Z"} + briefing := validValidationBriefing(metadata.GeneratedAt) + briefing.Email.SendPolicy = "automatic" + if err := validateBriefing(metadata, validationPublicationChanges{Status: "changed"}, briefing); err == nil { + t.Fatal("validateBriefing() accepted an automatic send policy") + } + briefing = validValidationBriefing(metadata.GeneratedAt) + briefing.Signals[0].DeltaTradeUSD = 999 + if err := validateBriefing(metadata, validationPublicationChanges{Status: "changed"}, briefing); err == nil { + t.Fatal("validateBriefing() accepted inconsistent signal arithmetic") + } +} + +func validValidationBriefing(generatedAt string) validationBriefing { + current := validationBriefingObservedValue{USATradeUSD: 120, ChinaTradeUSD: 80, TotalTradeUSD: 200, ChinaShare: 0.4} + previous := validationBriefingObservedValue{USATradeUSD: 80, ChinaTradeUSD: 80, TotalTradeUSD: 160, ChinaShare: 0.5} + ratio := 0.25 + signals := []validationBriefingSignal{ + {ID: "monthly-total-kor", Kind: "reporter_total_change", Title: "Korea total increased", Summary: "Summary", ReporterISO3: "KOR", ReporterName: "Korea, Rep.", Period: "2026-05", PreviousPeriod: "2026-04", Current: current, Previous: previous, DeltaTradeUSD: 40, ChangeRatio: &ratio, ChinaShareDelta: -0.1, Evidence: []string{"./semiconductors/monthly/KOR.json", "./semiconductors/monthly/index.json"}, Interpretation: "Boundary", MeasurementLimit: "Limit"}, + {ID: "anchor-share-kor", Kind: "anchor_share_shift", Title: "Korea share shifted", Summary: "Summary", ReporterISO3: "KOR", ReporterName: "Korea, Rep.", Period: "2026-05", PreviousPeriod: "2026-04", Current: current, Previous: previous, DeltaTradeUSD: 40, ChangeRatio: &ratio, ChinaShareDelta: -0.1, Evidence: []string{"./semiconductors/monthly/KOR.json", "./semiconductors/monthly/index.json"}, Interpretation: "Boundary", MeasurementLimit: "Limit"}, + {ID: "product-kor-854232", Kind: "product_total_change", Title: "Korea memories increased", Summary: "Summary", ReporterISO3: "KOR", ReporterName: "Korea, Rep.", Classification: "H6", Code: "854232", Label: "Memories", Period: "2026-05", PreviousPeriod: "2026-04", Current: current, Previous: previous, DeltaTradeUSD: 40, ChangeRatio: &ratio, ChinaShareDelta: -0.1, Evidence: []string{"./semiconductors/monthly/KOR.json", "./semiconductors/reference.json"}, Interpretation: "Boundary", MeasurementLimit: "Limit"}, + } + roles := []string{"cover", "scale", "anchor_balance", "product", "method", "cta"} + slides := make([]validationBriefingCarouselSlide, 0, len(roles)) + for index, role := range roles { + slides = append(slides, validationBriefingCarouselSlide{Order: index + 1, Role: role, Headline: "Headline", Body: "Body", Evidence: []string{"./semiconductors/monthly/index.json"}}) + } + return validationBriefing{ + SchemaVersion: "1.0", GeneratedAt: generatedAt, EditionID: "semiconductor-pulse-2026-05-20260717T000000Z", Status: "ready", + Title: "TradeGravity Semiconductor Pulse", Scope: "Scope", LatestPeriod: "2026-05", PreviousPeriod: "2026-04", PublicationStatus: "changed", ReviewRequired: true, + Signals: signals, + Email: validationBriefingEmail{Subject: "Subject", Preview: "Preview", Markdown: "[Evidence]({{BASE_URL}})", CTALabel: "Inspect", CTAPath: "./?tab=semiconductors", SendPolicy: "manual_review_required", PrimaryGoal: "Inspect evidence"}, + SocialCarousel: validationBriefingSocialCarousel{Format: "carousel_copy", AspectRatio: "4:5", ReviewPolicy: "manual_review_required", Slides: slides}, + Caveats: []string{"One", "Two", "Three"}, EvidenceEntryPoint: "./?tab=semiconductors", + } +} diff --git a/cmd/validator/extended.go b/cmd/validator/extended.go index ffcae3b..266dedd 100644 --- a/cmd/validator/extended.go +++ b/cmd/validator/extended.go @@ -379,6 +379,78 @@ type validationPublicationRevision struct { ChangeRatio *float64 `json:"change_ratio,omitempty"` } +type validationBriefing struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt string `json:"generated_at"` + EditionID string `json:"edition_id"` + Status string `json:"status"` + Title string `json:"title"` + Scope string `json:"scope"` + LatestPeriod string `json:"latest_period,omitempty"` + PreviousPeriod string `json:"previous_period,omitempty"` + PublicationStatus string `json:"publication_status"` + ReviewRequired bool `json:"review_required"` + Signals []validationBriefingSignal `json:"signals"` + Email validationBriefingEmail `json:"email"` + SocialCarousel validationBriefingSocialCarousel `json:"social_carousel"` + Caveats []string `json:"caveats"` + EvidenceEntryPoint string `json:"evidence_entry_point"` +} + +type validationBriefingSignal struct { + ID string `json:"id"` + Kind string `json:"kind"` + Title string `json:"title"` + Summary string `json:"summary"` + ReporterISO3 string `json:"reporter_iso3"` + ReporterName string `json:"reporter_name"` + Classification string `json:"classification,omitempty"` + Code string `json:"code,omitempty"` + Label string `json:"label,omitempty"` + Period string `json:"period"` + PreviousPeriod string `json:"previous_period"` + Current validationBriefingObservedValue `json:"current"` + Previous validationBriefingObservedValue `json:"previous"` + DeltaTradeUSD float64 `json:"delta_trade_usd"` + ChangeRatio *float64 `json:"change_ratio,omitempty"` + ChinaShareDelta float64 `json:"china_share_delta"` + Evidence []string `json:"evidence"` + Interpretation string `json:"interpretation"` + MeasurementLimit string `json:"measurement_limit"` +} + +type validationBriefingObservedValue struct { + USATradeUSD float64 `json:"usa_trade_usd"` + ChinaTradeUSD float64 `json:"china_trade_usd"` + TotalTradeUSD float64 `json:"total_trade_usd"` + ChinaShare float64 `json:"china_share"` +} + +type validationBriefingEmail struct { + Subject string `json:"subject"` + Preview string `json:"preview"` + Markdown string `json:"markdown"` + CTALabel string `json:"cta_label"` + CTAPath string `json:"cta_path"` + SendPolicy string `json:"send_policy"` + PrimaryGoal string `json:"primary_goal"` +} + +type validationBriefingSocialCarousel struct { + Format string `json:"format"` + AspectRatio string `json:"aspect_ratio"` + ReviewPolicy string `json:"review_policy"` + Slides []validationBriefingCarouselSlide `json:"slides"` +} + +type validationBriefingCarouselSlide struct { + Order int `json:"order"` + Role string `json:"role"` + Headline string `json:"headline"` + Body string `json:"body"` + Evidence []string `json:"evidence"` +} + type validationQuality struct { SchemaVersion string `json:"schema_version"` GeneratedAt string `json:"generated_at"` @@ -550,6 +622,13 @@ func validateExtendedDataset(dataDir string, metadata datasetMeta, latest datase if err := validatePublicationChanges(metadata, semiconductorMonthlyIndex, publicationChanges); err != nil { return err } + var briefing validationBriefing + if err := readJSON(filepath.Join(dataDir, "briefing.json"), &briefing); err != nil { + return fmt.Errorf("read briefing.json: %w", err) + } + if err := validateBriefing(metadata, publicationChanges, briefing); err != nil { + return err + } var tariffIndex validationTariffIndex if err := readJSON(filepath.Join(dataDir, "tariffs", "index.json"), &tariffIndex); err != nil { return fmt.Errorf("read tariff index: %w", err) @@ -575,7 +654,7 @@ func validateExtendedDataset(dataDir string, metadata datasetMeta, latest datase if err := readJSON(filepath.Join(dataDir, "catalog.json"), &catalog); err != nil { return fmt.Errorf("read catalog.json: %w", err) } - if err := validateCatalog(metadata, catalog, publicationChanges); err != nil { + if err := validateCatalog(metadata, catalog, publicationChanges, briefing); err != nil { return err } if err := validateExplanations(dataDir, metadata, latest); err != nil { @@ -588,7 +667,7 @@ func validateExtendedDataset(dataDir string, metadata datasetMeta, latest datase return validateContext(metadata, latest, contextData) } -func validateCatalog(metadata datasetMeta, catalog validationCatalog, publicationChanges validationPublicationChanges) error { +func validateCatalog(metadata datasetMeta, catalog validationCatalog, publicationChanges validationPublicationChanges, briefing validationBriefing) error { if catalog.SchemaVersion != "1.0" || catalog.GeneratedAt != metadata.GeneratedAt { return errorsForExtended("catalog provenance does not match metadata") } @@ -611,7 +690,7 @@ func validateCatalog(metadata datasetMeta, catalog validationCatalog, publicatio } seen[resource.ID] = resource } - for _, required := range []string{"headline_totals", "time_series", "country_context", "product_chapters", "quality", "strategic_hs6", "tariff_schedules", "bilateral_matrix", "semiconductor_atlas", "semiconductor_monthly", "publication_changes", "mirror_reconciliation", "scenario_runs"} { + for _, required := range []string{"headline_totals", "time_series", "country_context", "product_chapters", "quality", "strategic_hs6", "tariff_schedules", "bilateral_matrix", "semiconductor_atlas", "semiconductor_monthly", "publication_changes", "distribution_briefing", "mirror_reconciliation", "scenario_runs"} { if _, ok := seen[required]; !ok { return fmt.Errorf("catalog is missing resource %q", required) } @@ -668,6 +747,14 @@ func validateCatalog(metadata datasetMeta, catalog validationCatalog, publicatio if changesResource.Status != wantChangesStatus || changesResource.Provider != "tradegravity" || changesResource.ProductLevel != 6 || changesResource.Href != "./changes.json" { return errorsForExtended("catalog publication change resource does not match changes.json") } + briefingResource := seen["distribution_briefing"] + wantBriefingStatus := "partial" + if briefing.Status == "ready" { + wantBriefingStatus = "ready" + } + if briefingResource.Status != wantBriefingStatus || briefingResource.Provider != "tradegravity" || briefingResource.ProductLevel != 6 || briefingResource.Href != "./briefing.json" { + return errorsForExtended("catalog distribution briefing resource does not match briefing.json") + } return nil } @@ -741,6 +828,120 @@ func validatePublicationChanges(metadata datasetMeta, monthly validationSemicond return nil } +func validateBriefing(metadata datasetMeta, changes validationPublicationChanges, briefing validationBriefing) error { + if briefing.SchemaVersion != "1.0" || briefing.GeneratedAt != metadata.GeneratedAt || strings.TrimSpace(briefing.EditionID) == "" || strings.TrimSpace(briefing.Title) == "" || strings.TrimSpace(briefing.Scope) == "" { + return errorsForExtended("briefing provenance does not match metadata") + } + if briefing.Status != "ready" && briefing.Status != "unavailable" { + return fmt.Errorf("briefing has invalid status %q", briefing.Status) + } + if briefing.PublicationStatus != changes.Status || !briefing.ReviewRequired || len(briefing.Caveats) < 3 || !validBriefingHref(briefing.EvidenceEntryPoint) { + return errorsForExtended("briefing does not preserve publication status, review, caveat, or evidence requirements") + } + for _, caveat := range briefing.Caveats { + if strings.TrimSpace(caveat) == "" { + return errorsForExtended("briefing caveats must not be blank") + } + } + email := briefing.Email + if strings.TrimSpace(email.Subject) == "" || strings.TrimSpace(email.Preview) == "" || strings.TrimSpace(email.Markdown) == "" || strings.TrimSpace(email.CTALabel) == "" || !validBriefingHref(email.CTAPath) || email.SendPolicy != "manual_review_required" || strings.TrimSpace(email.PrimaryGoal) == "" { + return errorsForExtended("briefing email draft is incomplete or not review-gated") + } + carousel := briefing.SocialCarousel + if carousel.Format != "carousel_copy" || carousel.AspectRatio != "4:5" || carousel.ReviewPolicy != "manual_review_required" { + return errorsForExtended("briefing carousel contract is invalid") + } + if briefing.Status == "unavailable" { + if len(briefing.Signals) != 0 || len(carousel.Slides) != 0 { + return errorsForExtended("unavailable briefing must not publish signals or carousel slides") + } + return nil + } + if !monthPattern.MatchString(briefing.LatestPeriod) || !monthPattern.MatchString(briefing.PreviousPeriod) || briefing.LatestPeriod <= briefing.PreviousPeriod || !strings.Contains(email.Markdown, "{{BASE_URL}}") { + return errorsForExtended("ready briefing has invalid periods or unresolved delivery template") + } + wantKinds := []string{"reporter_total_change", "anchor_share_shift", "product_total_change"} + if len(briefing.Signals) != len(wantKinds) { + return fmt.Errorf("ready briefing has %d signals, want %d", len(briefing.Signals), len(wantKinds)) + } + seenIDs := make(map[string]struct{}, len(briefing.Signals)) + for index, signal := range briefing.Signals { + if signal.Kind != wantKinds[index] || strings.TrimSpace(signal.ID) == "" || strings.TrimSpace(signal.Title) == "" || strings.TrimSpace(signal.Summary) == "" || !iso3Pattern.MatchString(signal.ReporterISO3) || strings.TrimSpace(signal.ReporterName) == "" || !monthPattern.MatchString(signal.Period) || !monthPattern.MatchString(signal.PreviousPeriod) || signal.Period <= signal.PreviousPeriod || strings.TrimSpace(signal.Interpretation) == "" || strings.TrimSpace(signal.MeasurementLimit) == "" { + return fmt.Errorf("briefing has incomplete signal %+v", signal) + } + if _, exists := seenIDs[signal.ID]; exists { + return fmt.Errorf("briefing repeats signal id %q", signal.ID) + } + seenIDs[signal.ID] = struct{}{} + if signal.Kind == "product_total_change" { + if strings.TrimSpace(signal.Classification) == "" || !hs6Pattern.MatchString(signal.Code) || strings.TrimSpace(signal.Label) == "" { + return fmt.Errorf("briefing product signal is incomplete %+v", signal) + } + } else if signal.Classification != "" || signal.Code != "" || signal.Label != "" { + return fmt.Errorf("briefing aggregate signal claims product identity %+v", signal) + } + if err := validateBriefingObservedValue(signal.Current); err != nil { + return fmt.Errorf("briefing signal %s current value: %w", signal.ID, err) + } + if err := validateBriefingObservedValue(signal.Previous); err != nil { + return fmt.Errorf("briefing signal %s previous value: %w", signal.ID, err) + } + if !isFinite(signal.DeltaTradeUSD) || !approximatelyEqual(signal.DeltaTradeUSD, signal.Current.TotalTradeUSD-signal.Previous.TotalTradeUSD) || !isFinite(signal.ChinaShareDelta) || !approximatelyEqual(signal.ChinaShareDelta, signal.Current.ChinaShare-signal.Previous.ChinaShare) { + return fmt.Errorf("briefing signal %s has inconsistent deltas", signal.ID) + } + if signal.Previous.TotalTradeUSD > 0 { + if signal.ChangeRatio == nil || !isFinite(*signal.ChangeRatio) || !approximatelyEqual(*signal.ChangeRatio, signal.DeltaTradeUSD/signal.Previous.TotalTradeUSD) { + return fmt.Errorf("briefing signal %s has inconsistent change ratio", signal.ID) + } + } else if signal.ChangeRatio != nil { + return fmt.Errorf("briefing signal %s must omit a ratio with a zero baseline", signal.ID) + } + if len(signal.Evidence) < 2 { + return fmt.Errorf("briefing signal %s has insufficient evidence links", signal.ID) + } + for _, href := range signal.Evidence { + if !validBriefingHref(href) { + return fmt.Errorf("briefing signal %s has invalid evidence href %q", signal.ID, href) + } + } + } + if briefing.Signals[0].Period != briefing.LatestPeriod || briefing.Signals[0].PreviousPeriod != briefing.PreviousPeriod { + return errorsForExtended("briefing edition periods do not match the leading signal") + } + wantRoles := []string{"cover", "scale", "anchor_balance", "product", "method", "cta"} + if len(carousel.Slides) != len(wantRoles) { + return fmt.Errorf("briefing carousel has %d slides, want %d", len(carousel.Slides), len(wantRoles)) + } + for index, slide := range carousel.Slides { + if slide.Order != index+1 || slide.Role != wantRoles[index] || strings.TrimSpace(slide.Headline) == "" || strings.TrimSpace(slide.Body) == "" || len(slide.Evidence) == 0 { + return fmt.Errorf("briefing carousel has invalid slide %+v", slide) + } + for _, href := range slide.Evidence { + if !validBriefingHref(href) { + return fmt.Errorf("briefing carousel slide %d has invalid evidence href %q", slide.Order, href) + } + } + } + return nil +} + +func validateBriefingObservedValue(value validationBriefingObservedValue) error { + values := []float64{value.USATradeUSD, value.ChinaTradeUSD, value.TotalTradeUSD, value.ChinaShare} + for _, item := range values { + if !isFinite(item) || item < 0 { + return fmt.Errorf("contains invalid nonnegative value %v", item) + } + } + if value.ChinaShare > 1 || !approximatelyEqual(value.TotalTradeUSD, value.USATradeUSD+value.ChinaTradeUSD) || (value.TotalTradeUSD > 0 && !approximatelyEqual(value.ChinaShare, value.ChinaTradeUSD/value.TotalTradeUSD)) || (value.TotalTradeUSD == 0 && value.ChinaShare != 0) { + return errorsForExtended("totals or China share are inconsistent") + } + return nil +} + +func validBriefingHref(href string) bool { + return strings.HasPrefix(href, "./") && !strings.Contains(href, "..") && !strings.ContainsAny(href, "\r\n") +} + func sortedUnique(values []string) bool { if !sort.StringsAreSorted(values) { return false diff --git a/docs/DATA_SCHEMA.md b/docs/DATA_SCHEMA.md index 08e0587..eccf5a1 100644 --- a/docs/DATA_SCHEMA.md +++ b/docs/DATA_SCHEMA.md @@ -1,6 +1,6 @@ # Published data schema 2.0 -TradeGravity publishes one versioned artifact set under `site/data/`. Headline, product, strategic HS6, semiconductor, publish-to-publish change, tariff, bilateral-matrix, mirror-diagnostic, quality, and explanation artifacts share the trade publication timestamp. The semiconductor reference has its own additive schema and records both its editorial update date and publisher `generated_at`. `catalog.json` and `changes.json` have independent additive schemas and the same publisher timestamp. `context.json` has its own refresh time because it is built before the trade publisher. +TradeGravity publishes one versioned artifact set under `site/data/`. Headline, product, strategic HS6, semiconductor, publish-to-publish change, distribution briefing, tariff, bilateral-matrix, mirror-diagnostic, quality, and explanation artifacts share the trade publication timestamp. The semiconductor reference has its own additive schema and records both its editorial update date and publisher `generated_at`. `catalog.json`, `changes.json`, and `briefing.json` have independent additive schemas and the same publisher timestamp. `context.json` has its own refresh time because it is built before the trade publisher. ## Time and comparison semantics @@ -24,6 +24,7 @@ TradeGravity publishes one versioned artifact set under `site/data/`. Headline, | `semiconductors/monthly/index.json` | Focused monthly reporter/period discovery | UN Comtrade + semiconductor registry | | `semiconductors/monthly/{ISO3}.json` | Selected HS6 monthly USA/China flows | UN Comtrade | | `changes.json` | Previous-publication coverage, row, and value deltas for the focused monthly semiconductor layer | Publisher comparison of consecutive publications | +| `briefing.json` | Review-gated email and 4:5 carousel drafts derived from three cited monthly semiconductor signals | Deterministic publisher calculations over the focused monthly layer | | `tariffs/index.json` | Importer/year tariff partition discovery | WITS/TRAINS | | `tariffs/{ISO3}/{YEAR}.json` | Revision-aware strategic HS6 tariff rows | WITS/TRAINS | | `bilateral-matrix/index.json` | Multi-partner `TOTAL` partition discovery | UN Comtrade | @@ -39,7 +40,7 @@ TradeGravity publishes one versioned artifact set under `site/data/`. Headline, The catalog is the stable discovery layer for a dashboard that may grow beyond a few single-file datasets. Each resource declares an `id`, display title, `status`, analytical `grain`, `partitioning`, and an `href` only when an artifact is published. Current statuses are `ready`, `partial`, and `planned`. -`strategic_hs6`, `semiconductor_atlas`, `semiconductor_monthly`, `publication_changes`, `tariff_schedules`, `bilateral_matrix`, and `mirror_reconciliation` are published resources. The last ID is retained for catalog compatibility, but its title and artifact explicitly describe **unadjusted mirror-reporting diagnostics**, not a reconciled truth. Computed value-added networks and versioned scenario results remain planned contracts and do not claim that those observations exist. Published resources use relative same-origin paths; the validator rejects duplicate IDs, invalid statuses, unsafe paths, and metadata that conflicts with `meta.json`. +`strategic_hs6`, `semiconductor_atlas`, `semiconductor_monthly`, `publication_changes`, `distribution_briefing`, `tariff_schedules`, `bilateral_matrix`, and `mirror_reconciliation` are published resources. The last ID is retained for catalog compatibility, but its title and artifact explicitly describe **unadjusted mirror-reporting diagnostics**, not a reconciled truth. Computed value-added networks and versioned scenario results remain planned contracts and do not claim that those observations exist. Published resources use relative same-origin paths; the validator rejects duplicate IDs, invalid statuses, unsafe paths, and metadata that conflicts with `meta.json`. The current product resource demonstrates the intended scaling pattern: a small discovery index plus one reporter file. Higher-volume resources should use period, reporter, importer, industry, or sector chunks named in the catalog rather than expanding `latest.json`. @@ -209,6 +210,27 @@ Only mapped HS6 codes, monthly periods, and USA/CHN partners are admitted. The i `status` is `baseline` when no earlier comparable publication exists, `unchanged` when a previous publication exists but no admitted change is found, and `changed` otherwise. A baseline never masquerades as “no change.” The revision list is capped at 20 rows and ordered by the sum of absolute USA and China trade-value changes; global counts remain complete even when the list is truncated. Added and removed rows are counted but not represented as value revisions. The validator checks current-index identity, timestamps, dimensions, counts, finite values, revision arithmetic, and descending magnitude. +## Reviewed distribution briefing + +`briefing.json` uses schema `1.0` and converts the focused monthly semiconductor observations into one inspectable distribution contract. It contains exactly three deterministic signals when two comparable months exist: the largest absolute reporter total change, the largest absolute two-anchor China-share shift, and the largest absolute HS6 product change. Each signal retains current and previous USA/China values, arithmetic deltas, periods, interpretation limits, and relative evidence paths. + +The same signals feed two non-publishing draft formats: + +- `email` contains subject, preview, Markdown body, evidence CTA, `send_policy: "manual_review_required"`, and a primary goal; +- `social_carousel` contains six cited `4:5` copy slides with `review_policy: "manual_review_required"` in cover, scale, anchor-balance, product, method, and CTA order. + +The top-level `review_required` must remain `true`. A ready artifact is rejected if either channel permits automatic publication. If two comparable months are unavailable, the artifact fails closed with `status: "unavailable"`, no signals, and no sendable copy. The static application can download drafts and copy the evidence link; it does not maintain a subscriber database, send mail, or call a social publishing API. The offline distributor can render matched SVG and 1080×1350 PNG drafts from the validated slide model, but its manifest keeps both send and social authorization false. + +An optional local `approval.json` schema `1.0` is created only after the distributor verifies the exact manifest file set, byte counts, and SHA-256 digests. The manifest's carousel record includes the selected native `theme` (`intelligence-dark` or `editorial-light`) and fixed `caption_path: "carousel/caption.md"`. The caption is derived from the validated signals with period, evidence entry point, scope note, restrained tags, and a 1,800-rune project ceiling. A theme, caption, or image change alters hashes and invalidates an earlier approval. The approval records `scope: "content_release"`, the edition and manifest digest, reviewer, non-sensitive audience label, canonical UTC approval time, sorted channels, and fixed review attestations; automatic publishing remains false. + +An optional external `instagram-preflight.json` schema `1.0` binds an Instagram-approved manifest and approval digest to the theme, six-slide count, 1080×1350 dimensions, caption rune/hashtag counts, alt-text section count, and boolean integrity checks. It contains no caption text, hashtags, evidence URL, credentials, or account identity. `manual_upload_required` is true and `automatic_publish_authorized` is false. It must remain outside the approved kit; see [INSTAGRAM_PREFLIGHT.md](INSTAGRAM_PREFLIGHT.md). + +An optional local `delivery-preflight.json` schema `1.0` can then bind an email-approved edition to the SHA-256 digests of private subscriber and suppression CSVs. It records only aggregate consented, suppressed, suppression-row, and eligible counts; the approved audience label; the template, manifest, and approval digests; an explicit pilot ceiling; required unsubscribe headers and DKIM coverage; and whether individual opaque HTTPS unsubscribe URLs passed validation. It contains no recipient addresses, unsubscribe URLs, tokens, or local file paths. `provider_configured` and `delivery_authorized` remain false. The private CSV schemas and operational limits are documented in [EMAIL_DELIVERY_PREFLIGHT.md](EMAIL_DELIVERY_PREFLIGHT.md); neither the CSVs nor this local plan belongs under `site/data/`. + +The optional private subscription registry is deliberately outside the published schema. Its SQLite tables keep random subscription and pending-confirmation IDs, normalized email, audience, active/suppressed state, double-opt-in evidence, confirmation expiry/dispatch state, privacy-notice version, suppression reason/time, provider-feedback event IDs, and global bounced/complaint/provider-suppression state. Purpose-separated HMAC token payloads contain only a version and random ID. Confirmation GET is read-only and explicit POST records consent; globally suppressed addresses cannot receive confirmation or become active. Database files, WAL/SHM files, source/export CSVs, tokens, and secrets must never be published. See [UNSUBSCRIBE_SERVICE.md](UNSUBSCRIBE_SERVICE.md). + +An optional local `email-launch-authorization.json` schema `1.0` binds one Resend pilot to the exact preflight digest, edition, audience, sender, content/input digests, eligible count, operator, attestations, and a validity window of no more than one hour. It contains no recipient address or unsubscribe token. The accompanying private delivery-ledger SQLite schema stores only HMAC-derived recipient and delivery keys, aggregate labels, content/idempotency and authorization identifiers, provider message IDs, statuses, timestamps, and bounded non-sensitive reconciliation labels. Accepted and unresolved-pending rows both prevent an automatic resend. A provider-confirmed `not_accepted` resolution can transition back to a pending attempt only when a different launch-authorization digest is supplied later; the audit row and stable provider idempotency key are retained. Neither artifact is published, and neither changes the public `catalog.json`; see [EMAIL_PROVIDER_PILOT.md](EMAIL_PROVIDER_PILOT.md). + ## Multi-partner bilateral matrix The matrix index has `product_code: "TOTAL"`, `product_level: 0`, sorted reporter/partner/period dimensions, partition counts, partner-row counts, and source-observation counts. Each reporter/year file contains one row per alphabetic ISO3 partner: @@ -256,7 +278,7 @@ Run the same validation used before deployment: go run ./cmd/validator -dir site/data -min-reporters 40 ``` -The validator checks cross-file provenance and counts, reporter and period uniqueness, finite numbers, calculated totals/shares/balances, monthly product identities, mirror-pair arithmetic and disclosure, flow-availability identities, strategic registry membership, free/public reference policy, tariff rate identities, catalog contracts, context coverage, collection-run metadata, and every explanation citation. +The validator checks cross-file provenance and counts, reporter and period uniqueness, finite numbers, calculated totals/shares/balances, monthly product identities, briefing signal arithmetic and review gates, mirror-pair arithmetic and disclosure, flow-availability identities, strategic registry membership, free/public reference policy, tariff rate identities, catalog contracts, context coverage, collection-run metadata, and every explanation citation. ## CSV and filtered JSON diff --git a/docs/DISTRIBUTION.md b/docs/DISTRIBUTION.md new file mode 100644 index 0000000..380a713 --- /dev/null +++ b/docs/DISTRIBUTION.md @@ -0,0 +1,106 @@ +# Reviewed distribution workflow + +TradeGravity uses one evidence contract for its website briefing, email draft, and future social cards. The workflow is deliberately split into **analysis**, **rendering**, **editorial approval**, and **delivery** so that generating an asset never authorizes sending or publishing it. + +## Build a local kit + +Start from a validator-accepted `briefing.json` and the public URL at which its cited evidence will be available: + +```bash +go run ./cmd/distributor \ + -briefing examples/sample-data/briefing.json \ + -out distribution-kit \ + -base-url https://elecpapaya.github.io/TradeGravity/ \ + -theme intelligence-dark +``` + +`-theme` accepts `intelligence-dark` (the default analytical dark treatment) or `editorial-light` (a restrained light editorial treatment). Both are original, network-free native Go themes over the same validated six-slide model. Theme selection changes the PNG/SVG bytes and is recorded in the manifest, so changing it requires a new kit and approval. + +The command refuses unavailable or automatically publishable briefing contracts, insecure public base URLs, and an output directory that already exists. It performs no network request and sends nothing. + +The generated directory contains: + +```text +distribution-kit/ +├── manifest.json +├── REVIEW.md +├── email/ +│ ├── subject.txt +│ ├── preview.txt +│ ├── body.md +│ └── body.html +└── carousel/ + ├── index.html + ├── alt-text.md + ├── caption.md + ├── slide-01.svg … slide-06.svg + └── slide-01.png … slide-06.png +``` + +After review, the approval command adds `approval.json` beside the manifest. It is intentionally absent from a newly built kit. + +`carousel/caption.md` is a review-pending Instagram caption derived only from the three validated signals. It retains the comparison period, evidence entry point, scope warning, and restrained topic tags. The project imposes a 1,800-rune editorial ceiling. Edit the source briefing and regenerate instead of changing the caption independently. + +`manifest.json` records the edition, public evidence base, email CTA, selected native theme, caption path, 1080×1350 dimensions, available `png` and `svg` formats, review-pending state, explicit false send/publish authorization, and a byte count and SHA-256 digest for every reviewable file. The manifest is deterministic for the same briefing, base URL, and theme. + +## Editorial review + +Open `carousel/index.html` locally and preview `email/body.html` on desktop and mobile. Complete `REVIEW.md` before moving any file to an external provider. At minimum, verify: + +- all periods, values, directions, and source links against the cited JSON; +- month-to-month movement is not confused with a publish-to-publish revision; +- the email has one primary CTA back to the evidence; +- the six 4:5 cards remain legible at feed size and have reviewed alt text; and +- caveats do not become causal, routing, capacity, alignment, or investment claims. + +The SVG files are editable, resolution-independent originals. The matching PNG files are deterministic 1080×1350 raster assets generated from the same validated slide model; `carousel/index.html` deliberately previews those PNGs so the editor reviews the files intended for upload. Both formats and their SHA-256 hashes are created together, but they remain drafts. If copy or citations change, regenerate the whole kit instead of editing a PNG in place. Platform preview, alt-text entry, caption review, and the final Instagram publish action stay manual. + +The PNG renderer uses embedded Go fonts and makes no network request or external-font fetch. This keeps the output repeatable in CI and avoids silently changing typography between builds. Review every card at actual feed size because successful decoding and correct dimensions do not prove platform acceptance or reader comprehension. + +Both native palettes are tested across all three gradient stops. Normal text colors must retain at least 4.5:1 contrast, and the 20px bold role labels must retain at least 3:1 against their translucent pills. This automated contrast floor complements—rather than replaces—the feed-size visual and assistive-technology review. + +## Record content approval + +Complete `REVIEW.md`, then bind the unchanged manifest and all 20 tracked files to a named reviewer, non-sensitive audience label, channel list, and explicit UTC time: + +```bash +go run ./cmd/distribution-approval \ + -kit distribution-kit \ + -reviewer elecpapaya \ + -audience consented-internal-pilot \ + -channels email,instagram \ + -approved-at 2026-07-17T12:00:00Z \ + -attest-reviewed +``` + +The command fails if any tracked file was changed or removed, an untracked file was added, the manifest no longer has its review-pending/false-authorization gates, a channel is unsupported, or `approval.json` already exists. Rebuild the entire kit to make a new approval. Never place recipient addresses or provider secrets in the kit or audience label. + +`approval.json` has `scope: "content_release"` and binds the manifest's SHA-256 digest, file count, edition, reviewer, audience label, approved channels, time, and fixed attestations. It deliberately keeps `provider_delivery_ready`, `subscriber_consent_ready`, and `automatic_publish_ready` false. A future delivery adapter can call the same verifier and require the intended channel, but must enforce its own consent, suppression, sender, and provider gates. + +The approval record proves consistency with the local manifest; it does not authenticate the human reviewer by itself. Preserve it in a protected commit, release artifact, or separately signed record if reviewer authenticity is required. + +## Validate the manual Instagram package + +After an Instagram-channel approval, run `cmd/instagram-preflight` to recheck the unchanged manifest, six PNG dimensions, caption evidence/scope/tags, and all six alt-text sections. The aggregate output remains outside the kit and cannot publish anything. See [Instagram manual-publish preflight](INSTAGRAM_PREFLIGHT.md). + +## Manual GitHub Actions build + +The **Build reviewed distribution kit** workflow lets the operator choose either supported native theme, reads the validator-accepted `gh-pages` publication, builds the kit, verifies the theme and false social authorization in the manifest, and uploads it as a 14-day Actions artifact. It is manually dispatched and has read-only repository permissions. It does not have an email token, subscriber list, or social credential. + +## Delivery gate + +Provider-backed sending remains a separate operator-controlled step. Before enabling it, document and test: + +- double opt-in and the exact subscription promise; +- one-click unsubscribe and a durable suppression list; +- sender identity, SPF, DKIM, and DMARC; +- bounce and complaint handling; +- privacy notice, retention, deletion, and data-processing terms; +- secret storage outside the browser and generated artifact; and +- an approval record that binds one reviewed edition and its final hashes to one channel and audience. + +The content-release record implements only the editorial part of the last item. Run the provider-neutral [email delivery preflight](EMAIL_DELIVERY_PREFLIGHT.md) locally to validate a private double-opt-in CSV, apply a private suppression CSV, and create an aggregate-only plan. The preflight keeps provider configuration and delivery authorization false. + +The [provider-backed email pilot](EMAIL_PROVIDER_PILOT.md) adds a second, short-lived launch approval plus a fail-closed Resend adapter. It replays the exact preflight inputs immediately before sending and records accepted or uncertain attempts in a private HMAC-keyed SQLite ledger. Merely building, approving, or preflighting a kit still cannot send email; the live command additionally requires the matching authorization, provider and ledger secrets, and `-send-live`. + +TradeGravity should not use open tracking pixels by default. If aggregate link measurement is later added, disclose it and keep raw subscriber behavior out of the public dataset. diff --git a/docs/EMAIL_DELIVERY_PREFLIGHT.md b/docs/EMAIL_DELIVERY_PREFLIGHT.md new file mode 100644 index 0000000..cee97bd --- /dev/null +++ b/docs/EMAIL_DELIVERY_PREFLIGHT.md @@ -0,0 +1,79 @@ +# Email delivery preflight + +TradeGravity keeps subscriber addresses outside the repository, static site, distribution kit, approval record, and aggregate preflight artifact. The preflight CLI is a provider-neutral gate between reviewed content and any future network sender. It validates double opt-in evidence, applies a durable suppression list, enforces a small pilot ceiling, and then emits only counts and source-file digests. It sends no email. + +## Inputs + +Start with a distribution kit whose `approval.json` includes the `email` channel. Keep both CSV files in a private local directory outside the kit. + +The subscriber CSV header must be exactly: + +```csv +email,audience,status,consented_at,consent_method,consent_source,privacy_notice_version,unsubscribe_url +reader@example.invalid,consented-internal-pilot,active,2026-07-10T01:00:00Z,double_opt_in,website-form,v1,https://subscriptions.example.invalid/u/opaque-recipient-token +``` + +Every subscriber row must: + +- match the non-sensitive audience label in `approval.json`; +- have `status` equal to `active` and `consent_method` equal to `double_opt_in`; +- contain an RFC 3339 consent time that is not in the future; +- record a consent source and privacy-notice version; and +- use a unique plain email addr-spec without a display name; and +- provide a unique absolute HTTPS unsubscribe URL containing an opaque recipient token, with no credentials, fragment, or decoded email address. + +The suppression CSV header must be exactly: + +```csv +email,reason,suppressed_at +former-reader@example.invalid,unsubscribed,2026-07-12T03:00:00Z +``` + +Allowed reasons are `unsubscribed`, `bounced`, `complaint`, `invalid`, and `manual`. An empty suppression list still needs the header. Suppression always wins over active consent. + +## Run the preflight + +```bash +go run ./cmd/distribution-preflight \ + -kit distribution-kit \ + -subscribers private/subscribers.csv \ + -suppressions private/suppressions.csv \ + -out delivery-preflight.json \ + -generated-at 2026-07-17T12:30:00Z \ + -max-recipients 25 +``` + +The command refuses: + +- a kit without a valid email content approval; +- changed, missing, or untracked kit files; +- subscriber or suppression inputs stored inside the kit; +- malformed, duplicate, future-dated, wrong-audience, or non-double-opt-in subscriber rows; +- malformed or duplicate suppression rows; +- an empty post-suppression audience; and +- an eligible audience above the explicit pilot limit. + +`delivery-preflight.json` uses file mode `0600` where the operating system supports it and refuses to overwrite an existing file. It records the edition, manifest and approval digests, audience label, subscriber/suppression file digests, aggregate counts, template digest, required unsubscribe headers, and the pilot limit. It contains no recipient addresses, unsubscribe URLs, tokens, or local source paths. + +## What remains intentionally false + +A passing plan has: + +```json +{ + "consent_validated": true, + "suppression_applied": true, + "unsubscribe_urls_validated": true, + "contains_recipient_addresses": false, + "provider_configured": false, + "delivery_authorized": false +} +``` + +The approved HTML and Markdown templates contain exactly one `{{UNSUBSCRIBE_URL}}` placeholder in addition to the single primary evidence CTA. A future sender must replace that placeholder with a recipient-specific HTTPS unsubscribe URL, set `List-Unsubscribe` to that HTTPS URI, set `List-Unsubscribe-Post` to `List-Unsubscribe=One-Click`, and ensure a valid DKIM signature covers both headers. These are the one-click sender requirements defined by [RFC 8058](https://www.rfc-editor.org/rfc/rfc8058.html); Gmail's current sender guidance also requires a visible body link for subscribed messages and one-click support for high-volume senders. ([Gmail sender guidelines](https://support.google.com/mail/answer/81126)) + +The reference [provider-backed email pilot](EMAIL_PROVIDER_PILOT.md) re-runs the same in-memory preflight immediately before sending, requires unchanged source digests, and obtains a separate short-lived launch authorization. The operator must still authenticate the sender domain and deploy/test the signed Resend feedback handler so bounce, complaint, and provider-suppression events update the private registry before issuing that authorization. + +For a provider-neutral implementation that generates these URLs and exports both private CSVs, see the [subscription registry and unsubscribe service](UNSUBSCRIBE_SERVICE.md). It imports existing double-opt-in evidence; it does not create or prove the original opt-in. + +Do not upload the CSV inputs to GitHub Actions artifacts. Do not print the eligible in-memory address list, embed it in logs, or copy it into the aggregate JSON. A passing preflight is never a live-send authorization. diff --git a/docs/EMAIL_PROVIDER_PILOT.md b/docs/EMAIL_PROVIDER_PILOT.md new file mode 100644 index 0000000..f90345f --- /dev/null +++ b/docs/EMAIL_PROVIDER_PILOT.md @@ -0,0 +1,121 @@ +# Provider-backed email pilot + +TradeGravity includes a bounded Resend adapter for a deliberately small, consented email pilot. It is not connected to the static site or GitHub Actions and it does not run on a schedule. A live provider request is possible only when the reviewed kit, private subscriber and suppression inputs, aggregate preflight, short-lived launch authorization, provider API key, delivery-ledger secret, and explicit `-send-live` acknowledgement all agree. + +The adapter follows one-email/one-job lifecycle guidance: each semiconductor brief has one primary evidence CTA, one visible unsubscribe link, and no tracking pixel. It sends one provider request per recipient rather than placing multiple readers in `To`, `Cc`, or `Bcc`. + +Audience consent may be imported from an existing verified double-opt-in source or collected by the default-off signup/confirmation flow documented in [UNSUBSCRIBE_SERVICE.md](UNSUBSCRIBE_SERVICE.md). Confirmation mail is transactional and does not itself add an address to a briefing audience; only the confirmation-page POST does so. + +## Security boundary + +Keep these files and values outside the repository, distribution kit, public site, CI logs, and Actions artifacts: + +- subscriber and suppression CSVs; +- `delivery-preflight.json` and `email-launch-authorization.json`; +- `delivery-ledger.db` plus its WAL/SHM files and backups; +- `RESEND_API_KEY`; and +- `TRADEGRAVITY_DELIVERY_SECRET`, a stable random value of at least 32 bytes. + +The delivery ledger stores edition and audience labels, content digests, provider/idempotency identifiers, times, and HMAC-derived recipient keys. It does not store recipient addresses, unsubscribe URLs, or rendered bodies. File mode is requested as `0600` where supported, but the operator still needs encrypted storage, access control, backups, retention, and deletion procedures. + +The Resend endpoint is compiled as `https://api.resend.com/emails`; it cannot be redirected with a CLI flag that might exfiltrate the API key. HTTP timeouts and response-size limits are enforced, and provider error bodies are not printed. + +## 1. Produce the aggregate preflight + +Follow [Email delivery preflight](EMAIL_DELIVERY_PREFLIGHT.md) and keep the resulting files private: + +```bash +go run ./cmd/distribution-preflight \ + -kit distribution-kit \ + -subscribers private/subscribers.csv \ + -suppressions private/suppressions.csv \ + -out private/delivery-preflight.json \ + -generated-at 2026-07-17T12:30:00Z \ + -max-recipients 25 +``` + +## 2. Record a short-lived launch approval + +Verify the Resend sender domain, SPF, DKIM, and DMARC; deploy and test both the HTTPS unsubscribe endpoint and the signed feedback endpoint described in [UNSUBSCRIBE_SERVICE.md](UNSUBSCRIBE_SERVICE.md); prove that `email.bounced`, `email.complained`, and `email.suppressed` events become durable suppressions; review privacy controls; and inspect the final eligible pilot list. Then create an authorization that expires within one hour: + +```bash +go run ./cmd/email-launch-approval \ + -preflight private/delivery-preflight.json \ + -out private/email-launch-authorization.json \ + -provider resend \ + -from 'TradeGravityDelivery is not configured. This static build creates manually reviewed drafts and does not collect subscriber data or publish directly to social platforms.
+