diff --git a/.gitignore b/.gitignore index b6d67d4..25a2a9a 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ vendor/ # root-level binaries from `go build ./cmd/...` /broadsheet /broadsheet-server + +# Claude Code worktrees, scratch, and session state +/.claude/ diff --git a/cmd/broadsheet-server/ui.go b/cmd/broadsheet-server/ui.go index 3000917..ae97ced 100644 --- a/cmd/broadsheet-server/ui.go +++ b/cmd/broadsheet-server/ui.go @@ -389,7 +389,13 @@ func handleUIArchive(p *broadsheet.Engine) http.HandlerFunc { for _, d := range ds { have[d.UTC().Format("20060102")] = true } + // Prefer the live catalog name; for a paper dropped from the catalog, + // fall back to the name its archive was collected under; only then the + // bare id. name := names[id] + if name == "" { + name = p.ArchiveName(id) + } if name == "" { name = id } diff --git a/docs/architecture-review.md b/docs/architecture-review.md new file mode 100644 index 0000000..72b838a --- /dev/null +++ b/docs/architecture-review.md @@ -0,0 +1,166 @@ +# Architecture review — provider / catalog / database / archive boundaries + +**Date:** 2026-07-14 · **Status:** diagnosis recorded; no structural changes made yet (deliberate, pre-1.0, single user). + +This is a design-review snapshot, not a spec. It records deficiencies found in an +adversarial audit of the four-layer model (six attacker lenses → prosecutor +verification against code → defender steelman → this synthesis) so we can +course-correct deliberately when we choose to. Line references are to the tree as +of this date and will drift — treat them as pointers, not coordinates. + +## Verdict + +**The four-layer model is fundamentally sound.** The recent decisions — archive as +the source of truth for "what I have," a self-describing `.meta.json` sidecar for +portability, and two independent lifetimes (catalog membership governs *polling*; +archive retention governs *addressability*) — are correct and should be preserved. + +The deficiencies are real but cluster into **four root causes**, and the biggest one +(the split-ownership `sources` row) is the source of most of the reconcile churn. +None require a rewrite; the highest-leverage fix is a moderate, self-contained +refactor. + +## Root causes + +### RC1 — The split-ownership `sources` row (catalog ↔ store). *The central one.* +One SQLite row is co-owned *column by column*: the catalog owns +id/display_name/provider_type/provider_config/position (re-asserted every boot via +`INSERT … ON CONFLICT(id) DO UPDATE`, `store.go` seed), the store owns only +`enabled` (`SetSourceEnabled` is its sole writer; the `Enabled` in the seed row is +just the catalog default and is deliberately omitted from the UPDATE). **Nobody owns +the row.** The `sources` table is a near-total *mirror* of the catalog whose only +real justification is that `enabled` needed a home. + +This one decision radiates into: +- the upsert-except-`enabled` dance; +- the prune-cascade that deletes user state when a paper leaves the catalog (and on a downgrade to an older binary); +- the display-name **triple**-storage (see RC-adjacent below); +- `Location` being the one catalog column *not* mirrored, so `Catalog()` re-joins it from `catalog.All()` at call time — a read already split across two sources; +- the "next user-writable column a contributor adds (poll interval, crop) gets clobbered to the catalog default every boot" trap; +- DB-loss resets every `enabled` toggle to the catalog default. + +**Fix (recommended when we act):** stop mirroring the catalog into SQLite. Replace +`sources` with a small `source_state(id, enabled, position, …)` table holding *only* +user state. A "paper" = catalog entry (name/provider/config/location read directly +from the in-memory `catalog.All()`) ⋈ user override ⋈ archive. Cheaper here than in +a typical system because the catalog is embedded in-memory data — "joining" it is a +map lookup, not a SQL join. This also drops the display-name storage from triple to +the intended *double* (catalog live-name + sidecar durable-name), cleaning up the +Path C work rather than conflicting with it. + +### RC2 — Two persistence substrates, no spanning owner (store ↔ archive). +SQLite records and filesystem artifacts have independent lifetimes +(catalog-membership prune vs age-based retention) with no transaction or shared +"membership" abstraction bridging them. Consequences: +- **Non-atomic dual-write:** the reconciler `Put`s the archive then writes versions/health to the store with no spanning transaction (`reconcile.go`); a crash between them yields an edition on disk with no recorded version (harmless-ish: re-fetched next poll) — but it's unowned. +- **Four-plus divergent "which papers" sets** with none authoritative: `knownSource` unions live-set ∪ `archive.Newest` ∪ store-rows; `ArchiveIndex` uses archive dirs only; `Catalog()` uses store rows only; `RenderFor` checks *only* the live set (so it can `ErrUnknownSource` a paper that `knownSource` says is addressable). This divergence is the edge-case *generator* — most of this conversation was patching instances of it. +- **Crop in the wrong layer:** crop is rendering metadata *about archived bytes* but is keyed in the catalog-pruned store, so a dropped-but-archived paper would render without its crop and a re-add loses it. +- **Three uncoordinated retention passes** (`archive.Prune`, `pruneCache`, `store.PruneFetchEvents`) off one cutoff — low impact (all re-derivable), but three walks that can disagree mid-run. + +**Fix direction:** name ONE authority for "what I have / is renderable" — the archive +on disk — and route every read path through a single addressability resolver so the +sets stop diverging. Fold the retention passes. + +### RC3 — `id` is an accident, not a validated concept (cross-cutting). +A bare string is copied verbatim into: store PK, `provider_versions`/`fetch_events`/ +`crop_overrides` FK-by-convention, archive + cache directory names, URL path param, +and ETag/header — with **no owning constructor and no grammar**. `archive.SourceIDs` +returns every directory name unfiltered and feeds it into `filepath.Join`. This +underlies the `bra^pe-jdc` malformation, phantom addressable papers from +hand-dropped junk dirs, the "not user input" `gosec` nolint, and the +convention-only cross-table integrity. + +**Fix:** a `SourceID` type with one validating constructor (grammar: +`[a-z0-9-]+` or similar), enforced at catalog load and at archive-dir enumeration. +Small, high-value, and a **prerequisite for safely opening acquisition** (RC4b). + +> **Refuted (do not re-raise):** the "path-traversal RCE" via `/paper/..%2F..%2F…`. +> chi v5 delivers the still-encoded segment verbatim; `filepath.Join(root, "..%2F…")` +> stays inside root and just `ENOENT`s. chi won't put a raw `/` in a `{id}` segment. +> This is a hygiene/correctness defect (RC3), **not** an exploit. + +### RC4 — Boundaries shipped ahead of behavior; asymmetric openness. +- **4a — Dead crop schema.** `crop_overrides` is created, CHECK-constrained, and + cascade-DELETEd on prune, and `CropHints.MastheadText` is threaded + catalog→registry→`source.Source` — yet there is **no read/write path for crop + anywhere on main**. A contract (and a data-loss cascade) asserted for a payload of + nothing. *Note:* smart-crop is in-flight in a separate worktree and migrations are + append-only, so **don't delete the schema** — reserve it, and when crop lands put + it in `source_state`/sidecar (which fixes "crop in the wrong layer" for free). +- **4b — Asymmetric openness.** The archive is id-*open* (drop in a `/` dir and + it's named + renderable via `knownSource`, portability by design) but acquisition + is catalog-*closed* (`loadEnabled` only reads catalog-seeded rows; any non-catalog + id is a prune target; `catalog.All()` is embed-only; no add-source path). + Portability is half-built: you can *render* a transplanted paper but never *keep it + current*. See the forward plan below. + +## Confirmed deficiencies (condensed) + +| # | Deficiency | Boundary | Sev | Root | +|---|---|---|---|---| +| 1 | Split-ownership `sources` row; catalog mirror + `enabled` in one row | catalog-store | architectural | RC1 | +| 2 | Next user-writable column added to the upsert SET list gets clobbered every boot | catalog-store | architectural | RC1 | +| 3 | `crop_overrides` is dead schema (created + cascade-pruned, never read/written) | catalog-store | major | RC4a | +| 4 | Crop lives in the catalog-pruned store, not with the archived bytes it describes | store-archive | architectural | RC2/RC4a | +| 5 | Display name materialized in 3 places (catalog, store mirror, sidecar) | catalog-archive | minor | RC1 | +| 6 | `id` is an unvalidated stringly-typed join key across all four layers | cross-cutting | major | RC3 | +| 7 | 4+ divergent "which papers" sets; `RenderFor` bypasses `knownSource` | cross-cutting | architectural | RC2 | +| 8 | Reconciler dual-writes store + archive with no spanning transaction | store-archive | major | RC2 | +| 9 | Three uncoordinated retention passes over one cutoff | store-archive | minor | RC2 | +| 10 | Acquisition catalog-closed while archive is id-open (no user-source path) | provider-catalog | major | RC4b | +| 11 | `MediaType` round-trips lossily through the filename (Put discards `Edition.Media`, read re-derives from ext) | provider-archive | minor | RC2 | +| 12 | Provider/catalog config validated only at runtime, per-row, swallowed on failure | provider-catalog | minor | RC3 | +| 13 | No declared SQL foreign keys; integrity is a hand-written DELETE loop | store-store | minor | RC1 | +| 14 | `Config.Sources` embedder path forks control flow via scattered `if cfg.Sources != nil` | cross-cutting | minor | — | +| 15 | User intent (`enabled`, crop) lives only in SQLite; no portable backstop | store-archive | major | RC1/deferred | +| 16 | Archive silently caps at one edition per source per day (`dayUTC` + `` filename) | provider-archive | minor | RC2 | +| 17 | Asymmetric provider decode (freedomforum hand-rolled in registry vs wapo decoded into its struct) | provider-catalog | minor | RC3 | + +## Preserve — do not touch +Archive-as-truth + self-describing sidecar + portability; the two intentional +lifetimes (catalog→polling, archive→addressability); `SeedSources`' core intent +(catalog owns wiring, refreshed every boot; `enabled` never clobbered); the provider +abstraction with opaque, provider-owned version-token keys; the **fail-safe** +token-revert (a token for an edition that failed to store is reverted so the next +poll retries rather than 304-ing past a missing artifact); the `%PDF` sniff guard in +both providers; the render/serve caching correctness (singleflight cold-render +collapse, `renderSem`/`composeSem` bounds, mtime-stamped PNG invalidation). + +## Recommended course-correction (tiers, for when we act) +- **Tier 0 (root fix):** the `source_state` refactor (RC1) — dissolves RC1 and most of RC2's user-state issues; cleans up name-triplication. +- **Tier 1 (cheap, high-value):** `SourceID` validated type (RC3); seed-time catalog decode validation (fail-fast, not runtime-swallow); one addressability resolver so all read paths agree (RC2, incl. `RenderFor`). +- **Tier 2 (hygiene/coordinate):** fold the three retention passes; single-source `MediaType`; reserve/wire crop into `source_state`/sidecar with the smart-crop worktree; reconsider the embedder fork only if a `SourceProvider` seam earns its indirection. + +## Overreach risks — what NOT to do +- Don't split `sources` in a way that pushes name/position/**location** joins into every read site — route reads through `catalog.All()` (in-memory) + `source_state`, not a SQL join. (Cheap *here*; would be costly if the catalog were a DB table.) +- Don't build a heavyweight "membership authority" component — the divergence is really *two legitimate lifetimes + one leak* (`RenderFor` bypassing `knownSource`), not a missing framework. +- Don't move crop to the sidecar **and** keep a store copy "as an index" — that recreates the multi-writer sync problem we're criticizing for the display name. Pick one writer. +- Don't add `ON DELETE CASCADE` FKs *and* keep the explicit prune loop — the prune blast radius shouldn't depend on FK definitions. +- Don't delete the `crop_overrides` schema/threading while smart-crop is in-flight (append-only migrations; worktree conflict). +- Don't open a runtime user-add path *before* RC3 (SourceID) and fail-fast decode exist — it reintroduces exactly the swallowed-decode / unvalidated-id risks the closed catalog avoids by construction. + +## Forward plan — user-defined sources (RC4b), so we don't paint ourselves into a corner + +We may or may not build this, but the following keeps the door open. The eventual +shape: **acquisition becomes as open as the archive** — a user can define a source +(`id + name + provider type + provider config`) that isn't in the compile-time +catalog, so a transplanted/foreign paper can be kept current, not just rendered. + +**Target approach (when/if we do it):** user source *definitions* live in the data +dir (a `sources.json`, or a `user_sources` store table) and are **merged over** the +embedded catalog by a single seam. The registry validates each user entry +(`SourceID` grammar + provider `Decode`) at load, failing *loudly per-entry* (never +swallowed — this is now untrusted input). Reconcile/prune treat user-origin sources +as **never pruned by catalog-absence**. Adopting a transplanted archive = adding a +matching user source. + +**Corner-avoidance guidance for any work we do *before* then:** +1. **Model "the set of known source definitions" as a composable seam**, even while user sources are empty. When we do the `source_state` refactor, introduce a single `resolveSources()` (today: `return catalog.All()`) that every offer/reconcile/prune path goes through — so adding user sources later is *one* place, not a retrofit across N sites. +2. **Do `SourceID` validation (RC3) regardless** — it's a hard prerequisite for accepting user ids, and cheap/valuable on its own. +3. **Make provider-config decode fail-fast and validating (Tier 1)** — prerequisite for accepting user configs. +4. **Model source ORIGIN (catalog vs user) as a first-class attribute** the moment we touch the store/reconcile — so "not in the embedded catalog ⟹ prune" becomes "not in *any* known-source set ⟹ prune," and user sources are spared. This is the specific spot where continuing to deepen the "catalog is the only source-of-definition" assumption would paint us into a corner. +5. The archive's downstream openness (foreign ids render) is already the pattern; the plan just makes the upstream (acquisition) symmetric. + +**In short:** the two safe, corner-avoiding investments are `SourceID` validation and +a single `resolveSources()` seam. With those in place, user-defined sources is an +additive feature, not a refactor. diff --git a/docs/architecture.md b/docs/architecture.md index 07186aa..ce45fe7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -146,9 +146,38 @@ Sources live as *data* — a row in the store (and an entry in the embedded catalog): `provider_type` + a JSON config blob. The registry is the params-decoding seam that turns that data back into a typed provider value (`{"prefix": "NY_NYT"}` -> `FreedomForum{Prefix: "NY_NYT"}`), so the `Provider` -interface never changed. On first boot the store is seeded from the catalog -with the classic defaults enabled; embedders can still pass `Config.Sources` -to bypass the store entirely. +interface never changed. Embedders can still pass `Config.Sources` to bypass the +store entirely. + +Every boot, seeding *reconciles* the store to the catalog rather than just +filling it once. The split of ownership is the whole idea: the catalog owns a +paper's identity and wiring (display name, provider type + config, position); +the store owns the user's one choice, its enabled flag. So a new catalog paper +is inserted (at its catalog default), an existing paper's wiring is refreshed — +that is how a repointed provider or corrected config reaches installs that +already have the row — and a paper the catalog has dropped is pruned, all while +the user's enable/disable toggle is preserved. Pruning is safe precisely because +the store is wholly catalog-derived; the one exception is an explicit +`Config.Sources` engine, which has no store to reconcile. + +The **archive is independent of all this, and self-describing**. Catalog +membership governs polling and what the catalog UI offers; the archive is keyed +by source id and has its own age-based retention. A paper that's disabled — or +dropped from the catalog outright — keeps its collected front pages browsable and +renderable until they age out on the normal retention (`knownSource` treats an id +with editions on disk as addressable). Removing a paper never deletes data as a +side effect; the history just follows the same 14-day expiry as everything else. + +Each source directory also carries a tiny `.meta.json` sidecar (`{"name": …}`, +additive JSON) that the reconciler stamps as it archives. That makes the archive +*portable*: a `/` directory dropped into any install shows up named and +renderable with no catalog or store entry, because the browser resolves a name as +catalog → sidecar → bare id, and `knownSource`/`ArchiveIndex` key off the files on +disk. Labeling runs before the seed prune, so a paper dropped in the same release +keeps its name; the sidecar ages out with the directory. (The store row's display +name is the *rebuildable index*; the sidecar is the *durable identity* — the +beets/OCI-label pattern, chosen over keeping a retired store row so the archive +stands on its own.) ### Adding a provider @@ -156,6 +185,20 @@ Implement `Provider` in `internal/provider/` and return `Edition`s with th right `Media`. Nothing else changes — the engine archives, renders (per `Media`), prunes, and serves the same way regardless of provider. +`washingtonpost` is the second driver, and the first that isn't freedomforum. The +Post publishes its print edition as per-page PDFs on an open CloudFront CDN, keyed +by a full-date folder (`/20260713/A01_SU_EZ_DAILY_20260713.pdf`) rather than a +day-of-month. Two things make it different, both absorbed inside the provider: +the front-page filename carries a zone code that rotates day to day between a +small set (`SU`/`RE`) — the Post publishes exactly one per day, so a poll probes +the candidates and takes whichever exists — and a missing object typically comes +back as a `403` (S3 `AccessDenied`), though CloudFront may also 404. Because the +folder *is* the edition date, +that date is exact, so there's no `Last-Modified` guessing. (The upstream that +does list the exact URL, the "today's paper" HTML page, is Akamai bot-protected +and unreliable from a headless poller; the CDN itself is open and honors +conditional GET, so the provider talks to the CDN.) + ## The reconciler A background loop the server starts (the library doesn't — see [library vs diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 303bfaa..556ad8a 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -6,6 +6,7 @@ package archive import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -18,6 +19,21 @@ import ( const dateLayout = "20060102" +// metaFile is the per-source sidecar that makes the archive self-describing: it +// holds metadata captured while a source was archiving (currently just the +// display name) so a paper's history stays labeled with its real name even after +// it leaves the catalog and its store row is gone. JSON so fields can be added +// without a format change; the leading dot and non-date name keep it out of +// edition listings (list parses . and skips everything else). +const metaFile = ".meta.json" + +// sourceMeta is the archive's self-description for one source. Additive only — +// an older binary ignores unknown fields, a newer one defaults missing ones. +type sourceMeta struct { + // Name is the source's display name at the time it last archived. + Name string `json:"name,omitempty"` +} + // Store is an on-disk archive rooted at a directory. Layout: // // //. @@ -49,6 +65,22 @@ func mediaFromExt(e string) source.MediaType { } } +// containsEdition reports whether any directory entry is a date-named edition +// file (the sidecar label and write litter are not editions). Used by Prune to +// decide whether a source directory still holds anything worth keeping. +func containsEdition(entries []os.DirEntry) bool { + for _, f := range entries { + if f.IsDir() { + continue + } + name := f.Name() + if _, err := time.Parse(dateLayout, strings.TrimSuffix(name, filepath.Ext(name))); err == nil { + return true + } + } + return false +} + // Put writes an edition to the archive atomically and returns its entry. An // edition on a day we already hold is overwritten (a re-posted/corrected // edition wins). @@ -67,6 +99,50 @@ func (s *Store) Put(sourceID string, ed source.Edition) (Entry, error) { return Entry{SourceID: sourceID, Date: dayUTC(ed.Date), Media: ed.Media, Path: dst}, nil } +// SetName records a source's display name in its archive metadata, so the +// archive can label itself once the catalog no longer can. Idempotent; a blank +// id or name is a no-op. Read-modify-write preserves any other metadata fields. +// Written atomically like editions. +func (s *Store) SetName(sourceID, name string) error { + if sourceID == "" || name == "" { + return nil + } + dir := filepath.Join(s.Root, sourceID) + if err := os.MkdirAll(dir, 0o750); err != nil { + return fmt.Errorf("archive: mkdir %s: %w", dir, err) + } + m := s.meta(sourceID) + if m.Name == name { + return nil // already current — skip the rewrite + } + m.Name = name + b, err := json.Marshal(m) + if err != nil { + return fmt.Errorf("archive: marshal meta for %s: %w", sourceID, err) + } + return writeAtomic(filepath.Join(dir, metaFile), b) +} + +// Name returns the display name recorded for a source, or "" if none was ever +// written (a pre-metadata archive, or a source that never archived). +func (s *Store) Name(sourceID string) string { + return strings.TrimSpace(s.meta(sourceID).Name) +} + +// meta reads a source's archive metadata, returning the zero value when absent +// or unreadable (a pre-metadata archive is simply unlabeled, not an error). +func (s *Store) meta(sourceID string) sourceMeta { + b, err := os.ReadFile(filepath.Join(s.Root, sourceID, metaFile)) //nolint:gosec // G304: archive path rooted at s.Root with an internal source id, not user input + if err != nil { + return sourceMeta{} + } + var m sourceMeta + if err := json.Unmarshal(b, &m); err != nil { + return sourceMeta{} + } + return m +} + // Has reports whether an edition for (sourceID, date's day) is already stored. func (s *Store) Has(sourceID string, date time.Time) bool { if date.IsZero() { @@ -172,6 +248,16 @@ func (s *Store) Prune(retention time.Duration, now time.Time) (int, error) { } } } + // Reclaim a source directory once no editions remain — e.g. a paper the + // catalog dropped, whose editions have all aged out. Read the directory + // explicitly and skip on error: list() returns nil on ANY ReadDir failure, + // so keying off it alone could mistake a transient read error for "empty" + // and RemoveAll a still-populated directory. RemoveAll also clears the label + // and any write litter; an active source that Puts again re-creates the dir. + dir := filepath.Join(s.Root, d.Name()) + if entries, err := os.ReadDir(dir); err == nil && !containsEdition(entries) { + _ = os.RemoveAll(dir) + } } return removed, nil } diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go index 6de9ad9..5dbbb68 100644 --- a/internal/archive/archive_test.go +++ b/internal/archive/archive_test.go @@ -2,6 +2,7 @@ package archive import ( "os" + "path/filepath" "testing" "time" @@ -89,6 +90,57 @@ func TestPrune(t *testing.T) { } } +func TestPrune_ReclaimsEmptiedSourceDir(t *testing.T) { + s := &Store{Root: t.TempDir()} + // "gone" only holds an edition old enough to age out, plus a name label; + // "ny-nyt" keeps a recent one. After pruning, the emptied directory (label + // and all) is reclaimed and the live one stays. + _, _ = s.Put("gone", ed("2026-06-10", "old")) + _ = s.SetName("gone", "Gone Gazette") + _, _ = s.Put("ny-nyt", ed("2026-06-29", "recent")) + now := mustDay("2026-06-30") + + if _, err := s.Prune(14*24*time.Hour, now); err != nil { + t.Fatalf("Prune: %v", err) + } + if _, err := os.Stat(filepath.Join(s.Root, "gone")); !os.IsNotExist(err) { + t.Errorf("emptied source dir should be reclaimed; stat err = %v", err) + } + if got := s.Name("gone"); got != "" { + t.Errorf("label should be gone once editions age out, got %q", got) + } + if fi, err := os.Stat(filepath.Join(s.Root, "ny-nyt")); err != nil || !fi.IsDir() { + t.Errorf("active source dir must remain; err = %v", err) + } +} + +func TestName_RoundTripAndNotAnEdition(t *testing.T) { + s := &Store{Root: t.TempDir()} + if got := s.Name("x"); got != "" { + t.Errorf("Name of never-archived source = %q, want empty", got) + } + if _, err := s.Put("x", ed("2026-06-29", "front")); err != nil { + t.Fatalf("Put: %v", err) + } + if err := s.SetName("x", "The X Times"); err != nil { + t.Fatalf("SetName: %v", err) + } + if got := s.Name("x"); got != "The X Times" { + t.Errorf("Name = %q, want The X Times", got) + } + // The label must not be mistaken for an edition. + if eds := s.List("x"); len(eds) != 1 { + t.Errorf("List after SetName = %d editions, want 1 (label ignored)", len(eds)) + } + // A later rename overwrites in place. + if err := s.SetName("x", "X Herald"); err != nil { + t.Fatalf("SetName rename: %v", err) + } + if got := s.Name("x"); got != "X Herald" { + t.Errorf("Name after rename = %q, want X Herald", got) + } +} + func mustDay(s string) time.Time { d, _ := time.Parse("2006-01-02", s) return d.UTC() diff --git a/internal/catalog/catalog.json b/internal/catalog/catalog.json index 0c27cba..437cc78 100644 --- a/internal/catalog/catalog.json +++ b/internal/catalog/catalog.json @@ -67,15 +67,6 @@ }, "location": "Fairbanks, Alaska" }, - { - "id": "al-cch", - "name": "The Cherokee County Herald", - "provider": "freedomforum", - "config": { - "prefix": "AL_CCH" - }, - "location": "Centre, Alabama" - }, { "id": "al-dd", "name": "The Decatur Daily", @@ -238,15 +229,6 @@ }, "location": "Vorarlberg, Austria" }, - { - "id": "aut-sn", - "name": "Salzburger Nachrichten", - "provider": "freedomforum", - "config": { - "prefix": "AUT_SN" - }, - "location": "Salzburg, Austria" - }, { "id": "aut-vn", "name": "Vorarlberger Nachrichten", @@ -274,51 +256,6 @@ }, "location": "Phoenix, Arizona" }, - { - "id": "az-cvr", - "name": "Chino Valley Review", - "provider": "freedomforum", - "config": { - "prefix": "AZ_CVR" - }, - "location": "Chino Valley, Arizona" - }, - { - "id": "az-dc", - "name": "The Daily Courier", - "provider": "freedomforum", - "config": { - "prefix": "AZ_DC" - }, - "location": "Prescott, Arizona" - }, - { - "id": "az-kdm", - "name": "Kingman Miner", - "provider": "freedomforum", - "config": { - "prefix": "AZ_KDM" - }, - "location": "Kingman, Arizona" - }, - { - "id": "az-pvt", - "name": "Prescott Valley Tribune", - "provider": "freedomforum", - "config": { - "prefix": "AZ_PVT" - }, - "location": "Prescott Valley, Arizona" - }, - { - "id": "az-tnh", - "name": "Today's News-Herald", - "provider": "freedomforum", - "config": { - "prefix": "AZ_TNH" - }, - "location": "Lake Havasu City, Arizona" - }, { "id": "bel-dm", "name": "De Morgen", @@ -355,15 +292,6 @@ }, "location": "Itaja\u00ed, Brazil" }, - { - "id": "bra-dt", - "name": "Di\u00e1rio de Taubat\u00e9", - "provider": "freedomforum", - "config": { - "prefix": "BRA_DT" - }, - "location": "Taubat\u00e9, Brazil" - }, { "id": "bra-fdsp", "name": "Folha de S. Paulo", @@ -382,15 +310,6 @@ }, "location": "S\u00e3o Paulo, Brazil" }, - { - "id": "bra^pe-jdc", - "name": "Jornal do Commercio", - "provider": "freedomforum", - "config": { - "prefix": "BRA^PE_JDC" - }, - "location": "Recife, Brazil" - }, { "id": "ca-db", "name": "Daily Breeze", @@ -562,15 +481,6 @@ }, "location": "Sacramento, California" }, - { - "id": "ca-sc", - "name": "The Salinas Californian", - "provider": "freedomforum", - "config": { - "prefix": "CA_SC" - }, - "location": "Salinas, California" - }, { "id": "ca-sgvt", "name": "San Gabriel Valley Tribune", @@ -598,15 +508,6 @@ }, "location": "San Bernardino, California" }, - { - "id": "ca-tt", - "name": "The Tribune", - "provider": "freedomforum", - "config": { - "prefix": "CA_TT" - }, - "location": "San Luis Obispo, California" - }, { "id": "ca-vcs", "name": "Ventura County Star", @@ -715,15 +616,6 @@ }, "location": "Toronto, Canada" }, - { - "id": "can-tmg", - "name": "The Macleod Gazette", - "provider": "freedomforum", - "config": { - "prefix": "CAN_TMG" - }, - "location": "Fort Macleod, Canada" - }, { "id": "can-tp", "name": "The Province", @@ -958,15 +850,6 @@ }, "location": "Waterbury, Connecticut" }, - { - "id": "ct-rc", - "name": "The Register Citizen", - "provider": "freedomforum", - "config": { - "prefix": "CT_RC" - }, - "location": "Torrington, Connecticut" - }, { "id": "ct-rj", "name": "Record-Journal", @@ -1033,10 +916,8 @@ { "id": "dc-wp", "name": "The Washington Post", - "provider": "freedomforum", - "config": { - "prefix": "DC_WP" - }, + "provider": "washingtonpost", + "config": {}, "location": "Washington, District of Columbia" }, { @@ -1084,15 +965,6 @@ }, "location": "Suva, Fiji" }, - { - "id": "fl-bh", - "name": "The Bradenton Herald", - "provider": "freedomforum", - "config": { - "prefix": "FL_BH" - }, - "location": "Bradenton, Florida" - }, { "id": "fl-dbnj", "name": "The Daytona Beach News-Journal", @@ -1264,15 +1136,6 @@ }, "location": "Ft. Lauderdale, Florida" }, - { - "id": "fl-tb", - "name": "TBT*", - "provider": "freedomforum", - "config": { - "prefix": "FL_TB" - }, - "location": "St. Petersburg, Florida" - }, { "id": "fl-td", "name": "Tallahassee Democrat", @@ -1336,15 +1199,6 @@ }, "location": "Atlanta, Georgia" }, - { - "id": "ga-le", - "name": "Ledger-Enquirer", - "provider": "freedomforum", - "config": { - "prefix": "GA_LE" - }, - "location": "Columbus, Georgia" - }, { "id": "ga-rnt", "name": "Rome News-Tribune", @@ -1381,15 +1235,6 @@ }, "location": "Tbilisi, Georgia" }, - { - "id": "ger-aa", - "name": "Augsburger Allgemeine", - "provider": "freedomforum", - "config": { - "prefix": "GER_AA" - }, - "location": "Augsburg, Germany" - }, { "id": "ger-faz", "name": "Frankfurter Allgemeine Zeitung", @@ -1498,15 +1343,6 @@ }, "location": "Sioux City, Iowa" }, - { - "id": "ia-tg", - "name": "The Gazette", - "provider": "freedomforum", - "config": { - "prefix": "IA_TG" - }, - "location": "Cedar Rapids, Iowa" - }, { "id": "ia-tt", "name": "Ames Tribune", @@ -1516,15 +1352,6 @@ }, "location": "Ames, Iowa" }, - { - "id": "ia-wbt", - "name": "West Branch Times", - "provider": "freedomforum", - "config": { - "prefix": "IA_WBT" - }, - "location": "West Branch, Iowa" - }, { "id": "id-bcdb", "name": "Bonner County Daily Bee", @@ -1534,15 +1361,6 @@ }, "location": "Sandpoint, Idaho" }, - { - "id": "id-bfh", - "name": "Bonners Ferry Herald", - "provider": "freedomforum", - "config": { - "prefix": "ID_BFH" - }, - "location": "Bonners Ferry, Idaho" - }, { "id": "id-cdap", "name": "Coeur d\u2019Alene Press", @@ -1570,24 +1388,6 @@ }, "location": "Lewiston, Idaho" }, - { - "id": "id-mpdn", - "name": "Moscow-Pullman Daily News", - "provider": "freedomforum", - "config": { - "prefix": "ID_MPDN" - }, - "location": "Moscow, Idaho" - }, - { - "id": "id-snp", - "name": "Shoshone News-Press", - "provider": "freedomforum", - "config": { - "prefix": "ID_SNP" - }, - "location": "Osburn, Idaho" - }, { "id": "id-tn", "name": "Times-News", @@ -1651,15 +1451,6 @@ }, "location": "Danville, Illinois" }, - { - "id": "il-dh", - "name": "Daily Herald", - "provider": "freedomforum", - "config": { - "prefix": "IL_DH" - }, - "location": "Suburban Chicago, Illinois" - }, { "id": "il-dhr", "name": "Decatur Herald & Review", @@ -1777,15 +1568,6 @@ }, "location": "Springfield, Illinois" }, - { - "id": "il-th", - "name": "The Hinsdalean", - "provider": "freedomforum", - "config": { - "prefix": "IL_TH" - }, - "location": "Hinsdale, Illinois" - }, { "id": "il-tp", "name": "The Pantagraph", @@ -1831,15 +1613,6 @@ }, "location": "Bloomington, Indiana" }, - { - "id": "in-ids", - "name": "Indiana Daily Student", - "provider": "freedomforum", - "config": { - "prefix": "IN_IDS" - }, - "location": "Bloomington, Indiana" - }, { "id": "in-is", "name": "The Indianapolis Star", @@ -1858,15 +1631,6 @@ }, "location": "Lafayette, Indiana" }, - { - "id": "in-jg", - "name": "The Journal Gazette", - "provider": "freedomforum", - "config": { - "prefix": "IN_JG" - }, - "location": "Fort Wayne, Indiana" - }, { "id": "in-kt", "name": "Kokomo Tribune", @@ -2056,14 +1820,6 @@ }, "location": "Jerusalem, Israel" }, - { - "id": "isr-ya", - "name": "Yedioth Ahronoth", - "provider": "freedomforum", - "config": { - "prefix": "ISR_YA" - } - }, { "id": "ita-ls", "name": "La Stampa", @@ -2091,15 +1847,6 @@ }, "location": "Hutchinson, Kansas" }, - { - "id": "ks-ljw", - "name": "Lawrence Journal-World", - "provider": "freedomforum", - "config": { - "prefix": "KS_LJW" - }, - "location": "Lawrence, Kansas" - }, { "id": "ks-sj", "name": "Salina Journal", @@ -2262,15 +2009,6 @@ }, "location": "Alexandria, Louisiana" }, - { - "id": "lith-lr", - "name": "Lietuvos Rytas", - "provider": "freedomforum", - "config": { - "prefix": "LITH_LR" - }, - "location": "Vilnius, Lithuania" - }, { "id": "lux-lw", "name": "Luxemburger Wort", @@ -2406,15 +2144,6 @@ }, "location": "Kuala Lumpur, Malaysia" }, - { - "id": "md-cct", - "name": "Carroll County Times", - "provider": "freedomforum", - "config": { - "prefix": "MD_CCT" - }, - "location": "Westminster, Maryland" - }, { "id": "md-dt", "name": "The Daily Times", @@ -2433,24 +2162,6 @@ }, "location": "Hagerstown, Maryland" }, - { - "id": "md-tc", - "name": "The Capital", - "provider": "freedomforum", - "config": { - "prefix": "MD_TC" - }, - "location": "Annapolis, Maryland" - }, - { - "id": "md-ts", - "name": "The Baltimore Sun", - "provider": "freedomforum", - "config": { - "prefix": "MD_TS" - }, - "location": "Baltimore, Maryland" - }, { "id": "me-kj", "name": "Kennebec Journal", @@ -2766,24 +2477,6 @@ }, "location": "Port Huron, Michigan" }, - { - "id": "mn-bd", - "name": "Brainerd Dispatch", - "provider": "freedomforum", - "config": { - "prefix": "MN_BD" - }, - "location": "Brainerd, Minnesota" - }, - { - "id": "mn-bp", - "name": "Bemidji Pioneer", - "provider": "freedomforum", - "config": { - "prefix": "MN_BP" - }, - "location": "Bemidji, Minnesota" - }, { "id": "mn-pp", "name": "Pioneer Press", @@ -2811,15 +2504,6 @@ }, "location": "Minneapolis, Minnesota" }, - { - "id": "mn-wct", - "name": "West Central Tribune", - "provider": "freedomforum", - "config": { - "prefix": "MN_WCT" - }, - "location": "Willmar, Minnesota" - }, { "id": "mo-cdt", "name": "Columbia Daily Tribune", @@ -2856,15 +2540,6 @@ }, "location": "Springfield, Missouri" }, - { - "id": "mo-nnl", - "name": "Nodaway News Leader", - "provider": "freedomforum", - "config": { - "prefix": "MO_NNL" - }, - "location": "Maryville, Missouri" - }, { "id": "mo-slpd", "name": "St. Louis Post-Dispatch", @@ -2892,15 +2567,6 @@ }, "location": "Jackson, Mississippi" }, - { - "id": "ms-gc", - "name": "Greenwood Commonwealth", - "provider": "freedomforum", - "config": { - "prefix": "MS_GC" - }, - "location": "Greenwood, Mississippi" - }, { "id": "ms-ha", "name": "Hattiesburg American", @@ -3108,42 +2774,6 @@ }, "location": "Winston-Salem, North Carolina" }, - { - "id": "nd-dp", - "name": "The Dickinson Press", - "provider": "freedomforum", - "config": { - "prefix": "ND_DP" - }, - "location": "Dickinson, North Dakota" - }, - { - "id": "nd-gfh", - "name": "Grand Forks Herald", - "provider": "freedomforum", - "config": { - "prefix": "ND_GFH" - }, - "location": "Grand Forks, North Dakota" - }, - { - "id": "nd-js", - "name": "The Jamestown Sun", - "provider": "freedomforum", - "config": { - "prefix": "ND_JS" - }, - "location": "Jamestown, North Dakota" - }, - { - "id": "nd-tf", - "name": "The Forum", - "provider": "freedomforum", - "config": { - "prefix": "ND_TF" - }, - "location": "Fargo, North Dakota" - }, { "id": "ne-bds", "name": "Beatrice Daily Sun", @@ -3171,15 +2801,6 @@ }, "location": "Fremont, Nebraska" }, - { - "id": "ne-ljs", - "name": "Lincoln Journal Star", - "provider": "freedomforum", - "config": { - "prefix": "NE_LJS" - }, - "location": "Lincoln, Nebraska" - }, { "id": "ne-owh", "name": "Omaha World-Herald", @@ -3270,15 +2891,6 @@ }, "location": "Dover, New Hampshire" }, - { - "id": "nh-mlt", - "name": "Monadnock Ledger-Transcript", - "provider": "freedomforum", - "config": { - "prefix": "NH_MLT" - }, - "location": "Peterborough, New Hampshire" - }, { "id": "nh-ph", "name": "Portsmouth Herald", @@ -3405,15 +3017,6 @@ }, "location": "Trenton, New Jersey" }, - { - "id": "nm-dh", - "name": "The Deming Headlight", - "provider": "freedomforum", - "config": { - "prefix": "NM_DH" - }, - "location": "Deming, New Mexico" - }, { "id": "nm-lcsn", "name": "Las Cruces Sun-News", @@ -3468,15 +3071,6 @@ }, "location": "Reno, Nevada" }, - { - "id": "nv-sun", - "name": "Las Vegas Sun", - "provider": "freedomforum", - "config": { - "prefix": "NV_SUN" - }, - "location": "Las Vegas, Nevada" - }, { "id": "ny-bn", "name": "The Buffalo News", @@ -3738,24 +3332,6 @@ }, "location": "Cleveland, Ohio" }, - { - "id": "oh-ct", - "name": "The Chronicle-Telegram", - "provider": "freedomforum", - "config": { - "prefix": "OH_CT" - }, - "location": "Elyria, Ohio" - }, - { - "id": "oh-ddn", - "name": "Dayton Daily News", - "provider": "freedomforum", - "config": { - "prefix": "OH_DDN" - }, - "location": "Dayton, Ohio" - }, { "id": "oh-dr", "name": "The Daily Record", @@ -3765,15 +3341,6 @@ }, "location": "Wooster, Ohio" }, - { - "id": "oh-jn", - "name": "Journal News", - "provider": "freedomforum", - "config": { - "prefix": "OH_JN" - }, - "location": "Hamilton, Ohio" - }, { "id": "oh-leg", "name": "Lancaster Eagle-Gazette", @@ -3783,24 +3350,6 @@ }, "location": "Lancaster, Ohio" }, - { - "id": "oh-ln", - "name": "The Lima News", - "provider": "freedomforum", - "config": { - "prefix": "OH_LN" - }, - "location": "Lima, Ohio" - }, - { - "id": "oh-mg", - "name": "The Medina-Gazette", - "provider": "freedomforum", - "config": { - "prefix": "OH_MG" - }, - "location": "Medina, Ohio" - }, { "id": "oh-mnj", "name": "Mansfield News Journal", @@ -3864,15 +3413,6 @@ }, "location": "Canton, Ohio" }, - { - "id": "oh-sns", - "name": "Springfield News Sun", - "provider": "freedomforum", - "config": { - "prefix": "OH_SNS" - }, - "location": "Springfield, Ohio" - }, { "id": "oh-tb", "name": "The Blade", @@ -3936,15 +3476,6 @@ }, "location": "Tulsa, Oklahoma" }, - { - "id": "ok-wedn", - "name": "Weatherford Daily News", - "provider": "freedomforum", - "config": { - "prefix": "OK_WEDN" - }, - "location": "Weatherford, Oklahoma" - }, { "id": "or-rg", "name": "The Register-Guard", @@ -3981,15 +3512,6 @@ }, "location": "Altoona, Pennsylvania" }, - { - "id": "pa-cdt", - "name": "Centre Daily Times", - "provider": "freedomforum", - "config": { - "prefix": "PA_CDT" - }, - "location": "State College, Pennsylvania" - }, { "id": "pa-cpo", "name": "Chambersburg Public Opinion", @@ -4017,33 +3539,6 @@ }, "location": "Somerset, Pennsylvania" }, - { - "id": "pa-di", - "name": "The Daily Item", - "provider": "freedomforum", - "config": { - "prefix": "PA_DI" - }, - "location": "Sunbury, Pennsylvania" - }, - { - "id": "pa-egn", - "name": "Erie Gay News", - "provider": "freedomforum", - "config": { - "prefix": "PA_EGN" - }, - "location": "Erie, Pennsylvania" - }, - { - "id": "pa-et", - "name": "The Express-Times", - "provider": "freedomforum", - "config": { - "prefix": "PA_ET" - }, - "location": "Easton, Pennsylvania" - }, { "id": "pa-etn", "name": "Erie Times-News", @@ -4053,15 +3548,6 @@ }, "location": "Erie, Pennsylvania" }, - { - "id": "pa-her", - "name": "The Herald", - "provider": "freedomforum", - "config": { - "prefix": "PA_HER" - }, - "location": "Sharon, Pennsylvania" - }, { "id": "pa-ldn", "name": "Lebanon Daily News", @@ -4071,15 +3557,6 @@ }, "location": "Lebanon, Pennsylvania" }, - { - "id": "pa-lnp", - "name": "LNP", - "provider": "freedomforum", - "config": { - "prefix": "PA_LNP" - }, - "location": "Lancaster, Pennsylvania" - }, { "id": "pa-mc", "name": "The Morning Call", @@ -4188,15 +3665,6 @@ }, "location": "Waynesboro, Pennsylvania" }, - { - "id": "pa-yd", - "name": "York Dispatch", - "provider": "freedomforum", - "config": { - "prefix": "PA_YD" - }, - "location": "York, Pennsylvania" - }, { "id": "pa-ydr", "name": "York Daily Record", @@ -4233,24 +3701,6 @@ }, "location": "Manila, Philippines" }, - { - "id": "pol-kp", - "name": "Kurier Poranny", - "provider": "freedomforum", - "config": { - "prefix": "POL_KP" - }, - "location": "Bialystok, Poland" - }, - { - "id": "qatar-ts", - "name": "The Peninsula", - "provider": "freedomforum", - "config": { - "prefix": "QATAR_TS" - }, - "location": "Doha, Qatar" - }, { "id": "ri-ndn", "name": "Newport Daily News", @@ -4269,24 +3719,6 @@ }, "location": "Providence, Rhode Island" }, - { - "id": "ri-ws", - "name": "The Westerly Sun", - "provider": "freedomforum", - "config": { - "prefix": "RI_WS" - }, - "location": "Westerly, Rhode Island" - }, - { - "id": "rom-iz", - "name": "Informatia Zilei", - "provider": "freedomforum", - "config": { - "prefix": "ROM_IZ" - }, - "location": "Satu Mare, Romania" - }, { "id": "rus-mp", "name": "Moskovskaya Pravda", @@ -4304,14 +3736,6 @@ "prefix": "SAF_CT" } }, - { - "id": "saf-ti", - "name": "The Independent", - "provider": "freedomforum", - "config": { - "prefix": "SAF_TI" - } - }, { "id": "saf-tm", "name": "The Mercury", @@ -4411,15 +3835,6 @@ }, "location": "Aberdeen, South Dakota" }, - { - "id": "sd-dr", - "name": "Mitchell Republic", - "provider": "freedomforum", - "config": { - "prefix": "SD_DR" - }, - "location": "Mitchell, South Dakota" - }, { "id": "sd-rcj", "name": "Rapid City Journal", @@ -4537,15 +3952,6 @@ }, "location": "Lleida, Spain" }, - { - "id": "swe-dn", - "name": "Dagens Nyheter", - "provider": "freedomforum", - "config": { - "prefix": "SWE_DN" - }, - "location": "Stockholm, Sweden" - }, { "id": "swe-ex", "name": "Expressen", @@ -4573,15 +3979,6 @@ }, "location": "Malm\u00f6, Sweden" }, - { - "id": "swi-bk", - "name": "Blick", - "provider": "freedomforum", - "config": { - "prefix": "SWI_BK" - }, - "location": "Z\u00fcrich, Switzerland" - }, { "id": "taiw-tmt", "name": "The Merit Times", @@ -4618,15 +4015,6 @@ }, "location": "Murfreesboro, Tennessee" }, - { - "id": "tn-dt", - "name": "The Daily Times", - "provider": "freedomforum", - "config": { - "prefix": "TN_DT" - }, - "location": "Maryville, Tennessee" - }, { "id": "tn-js", "name": "The Jackson Sun", @@ -4699,15 +4087,6 @@ }, "location": "Istanbul, Turkey" }, - { - "id": "tur-ticar", - "name": "Ticaret Gazetesi", - "provider": "freedomforum", - "config": { - "prefix": "TUR_TICAR" - }, - "location": "Izmir, Turkey" - }, { "id": "tur-yen", "name": "Yeniasir", @@ -4762,15 +4141,6 @@ }, "location": "Corpus Christi, Texas" }, - { - "id": "tx-dmn", - "name": "The Dallas Morning News", - "provider": "freedomforum", - "config": { - "prefix": "TX_DMN" - }, - "location": "Dallas, Texas" - }, { "id": "tx-dmnhn", "name": "The Dallas Morning News", @@ -4789,15 +4159,6 @@ }, "location": "Denton, Texas" }, - { - "id": "tx-dv", - "name": "Dallas Voice", - "provider": "freedomforum", - "config": { - "prefix": "TX_DV" - }, - "location": "Dallas, Texas" - }, { "id": "tx-ept", "name": "El Paso Times", @@ -4852,15 +4213,6 @@ }, "location": "Lubbock, Texas" }, - { - "id": "tx-ldn", - "name": "The Lufkin Daily News", - "provider": "freedomforum", - "config": { - "prefix": "TX_LDN" - }, - "location": "Lufkin, Texas" - }, { "id": "tx-lmt", "name": "Laredo Morning Times", @@ -4969,15 +4321,6 @@ }, "location": "Waco, Texas" }, - { - "id": "uae-ab", - "name": "The Daily Al Bayan", - "provider": "freedomforum", - "config": { - "prefix": "UAE_AB" - }, - "location": "Dubai, United Arab Emirates" - }, { "id": "uae-gn", "name": "Gulf News", @@ -5023,15 +4366,6 @@ }, "location": "St. George, Utah" }, - { - "id": "va-ach", - "name": "Arlington Catholic Herald", - "provider": "freedomforum", - "config": { - "prefix": "VA_ACH" - }, - "location": "Arlington, Virginia" - }, { "id": "va-bhc", "name": "Bristol Herald Courier", @@ -5113,15 +4447,6 @@ }, "location": "Waynesboro, Virginia" }, - { - "id": "va-pol", - "name": "POLITICO", - "provider": "freedomforum", - "config": { - "prefix": "VA_POL" - }, - "location": "Arlington, Virginia" - }, { "id": "va-rtd", "name": "Richmond Times-Dispatch", @@ -5167,15 +4492,6 @@ }, "location": "Bellingham, Washington" }, - { - "id": "wa-cbh", - "name": "Columbia Basin Herald", - "provider": "freedomforum", - "config": { - "prefix": "WA_CBH" - }, - "location": "Moses Lake, Washington" - }, { "id": "wa-col", "name": "The Columbian", @@ -5266,15 +4582,6 @@ }, "location": "Chippewa Falls, Wisconsin" }, - { - "id": "wi-dcn", - "name": "Dunn County News", - "provider": "freedomforum", - "config": { - "prefix": "WI_DCN" - }, - "location": "Menomonie, Wisconsin" - }, { "id": "wi-dt", "name": "Daily Tribune", @@ -5302,15 +4609,6 @@ }, "location": "Manitowoc, Wisconsin" }, - { - "id": "wi-jg", - "name": "The Janesville Gazette", - "provider": "freedomforum", - "config": { - "prefix": "WI_JG" - }, - "location": "Janesville, Wisconsin" - }, { "id": "wi-mjs", "name": "Milwaukee Journal Sentinel", @@ -5356,15 +4654,6 @@ }, "location": "Portage, Wisconsin" }, - { - "id": "wi-sn", - "name": "The Star News", - "provider": "freedomforum", - "config": { - "prefix": "WI_SN" - }, - "location": "Medford, Wisconsin" - }, { "id": "wi-sp", "name": "The Sheboygan Press", @@ -5419,15 +4708,6 @@ }, "location": "New York, New York" }, - { - "id": "wv-pns", - "name": "Parkersburg News and Sentinel", - "provider": "freedomforum", - "config": { - "prefix": "WV_PNS" - }, - "location": "Parkersburg, West Virginia" - }, { "id": "wv-rh", "name": "The Register-Herald", @@ -5437,15 +4717,6 @@ }, "location": "Beckley, West Virginia" }, - { - "id": "wv-twv", - "name": "Times West Virginian", - "provider": "freedomforum", - "config": { - "prefix": "WV_TWV" - }, - "location": "Fairmont, West Virginia" - }, { "id": "wy-cst", "name": "Casper Star-Tribune", diff --git a/internal/provider/freedomforum/freedomforum.go b/internal/provider/freedomforum/freedomforum.go index 24b08f7..2b81aeb 100644 --- a/internal/provider/freedomforum/freedomforum.go +++ b/internal/provider/freedomforum/freedomforum.go @@ -82,6 +82,12 @@ func (f FreedomForum) Poll(ctx context.Context, deps source.Deps, seen map[strin case http.StatusNotFound: // Nothing there — drop any stale token so we re-probe cleanly. default: + // An unexpected status (typically an upstream 5xx) is a failed probe, + // not a determinate answer: count it toward the failure gate so an + // all-error poll surfaces as a failure instead of a silent healthy + // no-op, and retain the token so we retry next cycle. + probeErrs++ + lastErr = fmt.Errorf("freedomforum: %s returned unexpected status %d", url, status) if deps.Logger != nil { deps.Logger.Warn("freedomforum unexpected status", "url", url, "status", status) } diff --git a/internal/provider/washingtonpost/washingtonpost.go b/internal/provider/washingtonpost/washingtonpost.go new file mode 100644 index 0000000..03269e9 --- /dev/null +++ b/internal/provider/washingtonpost/washingtonpost.go @@ -0,0 +1,230 @@ +// Package washingtonpost implements source.Provider for The Washington Post's +// daily print edition, served as per-page PDFs from its CloudFront CDN. +// +// Unlike freedomforum, the Post is not a day-of-month archive: each edition +// lives in a full-date folder (YYYYMMDD) and the front page's filename carries a +// zone code that rotates day to day between a small set (observed: SU or RE), +// with the edition/product codes stable (EZ/DAILY). WaPo publishes exactly one +// of those zones per day, so a poll probes the zone candidates against the open +// CDN and takes whichever one exists — the "today's paper" HTML page that lists +// the exact URL is Akamai bot-protected and unreliable for a headless poller, +// while the CDN itself is open and honors conditional GET. Because the folder is +// the full edition date, that date is authoritative — there's no Last-Modified +// guessing. See docs/architecture.md. +package washingtonpost + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/kelchm/broadsheet/internal/buildinfo" + "github.com/kelchm/broadsheet/internal/source" +) + +// baseURL is the CloudFront distribution root. A package var (not a const) so +// tests can point the provider at an httptest server, mirroring freedomforum. +var baseURL = "https://dedq39jz5ilmb.cloudfront.net" + +// Filename defaults. Only the zone rotates across days; the rest have held +// constant across every observed edition (weekdays, weekends, and holidays +// alike). They are still overridable via config so a scheme tweak upstream can +// be absorbed by editing the catalog rather than shipping a new binary. +const ( + defaultPage = "A01" // front page (section A, page 1) + defaultEdition = "EZ" + defaultProduct = "DAILY" +) + +// defaultZones are the front-page zone codes to probe. WaPo publishes exactly +// one per day; which one rotates for reasons that don't track the weekday, so +// both are always tried. +var defaultZones = []string{"SU", "RE"} + +// WashingtonPost backs the Post's front page. The zero value is usable — Poll +// fills blank fields with the defaults above — and the fields are exported so +// the registry can decode a config override into them. +type WashingtonPost struct { + Page string `json:"page,omitempty"` + Zones []string `json:"zones,omitempty"` + Edition string `json:"edition,omitempty"` + Product string `json:"product,omitempty"` +} + +// New returns a fully-defaulted provider. Passing zero values selects the +// defaults for each field. +func New(page string, zones []string, edition, product string) WashingtonPost { + return WashingtonPost{Page: page, Zones: zones, Edition: edition, Product: product}.defaulted() +} + +// defaulted returns a copy with blank fields filled from the defaults. Keeping +// this the single source of defaults means a config that omits a field and a +// caller that constructs the zero value land on the same behavior. +func (w WashingtonPost) defaulted() WashingtonPost { + if w.Page == "" { + w.Page = defaultPage + } + if len(w.Zones) == 0 { + w.Zones = defaultZones + } + if w.Edition == "" { + w.Edition = defaultEdition + } + if w.Product == "" { + w.Product = defaultProduct + } + return w +} + +// url is the CDN URL for one day/zone, e.g. +// https:///20260713/A01_SU_EZ_DAILY_20260713.pdf. The date appears twice: +// as the folder and inside the filename, both the edition's calendar date. +func (w WashingtonPost) url(day time.Time, zone string) string { + d := day.UTC().Format("20060102") + return fmt.Sprintf("%s/%s/%s_%s_%s_%s_%s.pdf", baseURL, d, w.Page, zone, w.Edition, w.Product, d) +} + +// Poll probes each candidate (day, zone) for the current edition. The day window +// is UTC yesterday/today/tomorrow — the same belt-and-suspenders span +// freedomforum uses; here it covers the pre-dawn window before a fresh edition +// posts (~00:15 ET) plus any clock skew, since the Post's folder date is always +// UTC-today or UTC-yesterday. Each probe is a conditional GET keyed by the +// previously-seen ETag, so an unchanged file costs a 304 and a nonexistent one a +// 403 (S3 AccessDenied) or 404. +func (w WashingtonPost) Poll(ctx context.Context, deps source.Deps, seen map[string]string, now time.Time) ( + []source.Edition, map[string]string, error) { + + w = w.defaulted() + + client := deps.HTTP + if client == nil { + client = http.DefaultClient + } + + versions := make(map[string]string) + var editions []source.Edition + + deltas := []int{-1, 0, 1} // yesterday, today, tomorrow (UTC) + probes := 0 + probeErrs := 0 + var lastErr error + + for _, delta := range deltas { + day := now.UTC().AddDate(0, 0, delta) + for _, zone := range w.Zones { + url := w.url(day, zone) + probes++ + + ed, etag, status, err := fetchConditional(ctx, client, url, seen[url], day) + if err != nil { + // A transient error on one probe must not sink the others; keep the + // version we had so a later poll can still short-circuit. + probeErrs++ + lastErr = err + if deps.Logger != nil { + deps.Logger.Debug("washingtonpost probe failed", "url", url, "err", err) + } + if v, ok := seen[url]; ok { + versions[url] = v + } + continue + } + + switch status { + case http.StatusOK: + editions = append(editions, *ed) + versions[url] = etag + case http.StatusNotModified: + versions[url] = seen[url] // unchanged; retain the token + case http.StatusForbidden, http.StatusNotFound: + // Nothing there (S3 returns 403 for a missing key, CloudFront may + // 404) — drop any stale token so we re-probe cleanly. + default: + // An unexpected status (typically an upstream 5xx) is a failed + // probe, not a determinate answer: count it toward the failure gate + // so an all-error poll surfaces as a failure instead of a silent + // healthy no-op, and retain the token so we retry next cycle. + probeErrs++ + lastErr = fmt.Errorf("washingtonpost: %s returned unexpected status %d", url, status) + if deps.Logger != nil { + deps.Logger.Warn("washingtonpost unexpected status", "url", url, "status", status) + } + if v, ok := seen[url]; ok { + versions[url] = v + } + } + } + } + + // Only a hard failure — every probe failed at the transport level (upstream + // unreachable) — is an error. A mix of 200/304/403 is a healthy poll. + if probeErrs == probes && lastErr != nil { + return editions, versions, fmt.Errorf("washingtonpost: all probes failed: %w", lastErr) + } + return editions, versions, nil +} + +// fetchConditional issues a conditional GET. On 200 it returns the edition; on +// 304/403/404/other it returns a nil edition and the status for the caller to +// act on. etag is the token to send as If-None-Match (empty to skip). day is the +// folder date being probed, which is the edition's calendar date. +func fetchConditional(ctx context.Context, client *http.Client, url, etag string, day time.Time) ( + *source.Edition, string, int, error) { + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, "", 0, fmt.Errorf("washingtonpost: build request: %w", err) + } + req.Header.Set("User-Agent", buildinfo.UserAgent()) + if etag != "" { + req.Header.Set("If-None-Match", etag) + } + + resp, err := client.Do(req) + if err != nil { + return nil, "", 0, fmt.Errorf("washingtonpost: GET %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) // allow connection reuse + return nil, "", resp.StatusCode, nil + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, "", resp.StatusCode, fmt.Errorf("washingtonpost: read body: %w", err) + } + // A 200 whose body isn't a PDF (an HTML/XML error page slipping through with + // a 200, a captive portal, an empty body) must not become an edition — and + // must not burn the ETag, or the next poll 304s past the real file forever. + // Treat it as a probe error so the caller retains the old token and retries + // next cycle. Per the PDF spec's reader tolerance the header may sit anywhere + // in the first 1024 bytes, so sniff that window rather than byte 0 only. + head := data + if len(head) > 1024 { + head = head[:1024] + } + if !bytes.Contains(head, []byte("%PDF")) { + return nil, "", resp.StatusCode, fmt.Errorf("washingtonpost: %s returned %d bytes that are not a PDF", url, len(data)) + } + newETag := resp.Header.Get("ETag") + ed := &source.Edition{ + Date: editionDate(day), + Version: newETag, + Media: source.MediaPDF, + Data: data, + } + return ed, newETag, resp.StatusCode, nil +} + +// editionDate is the folder's calendar date at day precision in UTC. Unlike +// freedomforum, the Post's URL carries the full date, so this is exact rather +// than a Last-Modified reading — and never zero, which the archive rejects. +func editionDate(day time.Time) time.Time { + d := day.UTC() + return time.Date(d.Year(), d.Month(), d.Day(), 0, 0, 0, 0, time.UTC) +} diff --git a/internal/provider/washingtonpost/washingtonpost_test.go b/internal/provider/washingtonpost/washingtonpost_test.go new file mode 100644 index 0000000..3792143 --- /dev/null +++ b/internal/provider/washingtonpost/washingtonpost_test.go @@ -0,0 +1,283 @@ +package washingtonpost + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/kelchm/broadsheet/internal/source" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// keyOf turns a request path like /20260713/A01_SU_EZ_DAILY_20260713.pdf into +// the "20260713/SU" key the fake CDN is indexed by (date folder + zone). +func keyOf(path string) string { + segs := strings.Split(strings.Trim(path, "/"), "/") + if len(segs) != 2 { + return "" + } + date := segs[0] + parts := strings.Split(strings.TrimSuffix(segs[1], ".pdf"), "_") + if len(parts) < 2 { + return "" + } + return date + "/" + parts[1] // date + zone +} + +type canned struct { + etag string + body string +} + +// fakeCDN serves canned 200s for the given date/zone keys, a 403 (S3 +// AccessDenied, as the real CDN does for a missing key) otherwise, and honors +// If-None-Match with a 304. +func fakeCDN(byKey map[string]canned) *http.Client { + return &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + mk := func(status int, hdr http.Header, body string) *http.Response { + if hdr == nil { + hdr = http.Header{} + } + return &http.Response{ + StatusCode: status, Header: hdr, Request: r, + Body: io.NopCloser(strings.NewReader(body)), + } + } + c, ok := byKey[keyOf(r.URL.Path)] + if !ok { + return mk(http.StatusForbidden, nil, + `AccessDenied`), nil + } + if inm := r.Header.Get("If-None-Match"); inm != "" && inm == c.etag { + return mk(http.StatusNotModified, nil, ""), nil + } + h := http.Header{} + h.Set("ETag", c.etag) + return mk(http.StatusOK, h, c.body), nil + })} +} + +// now is a fixed Monday noon UTC; the probe window is {07-12, 07-13, 07-14}. +var now = time.Date(2026, 7, 13, 12, 0, 0, 0, time.UTC) + +func TestPoll_ColdPicksWhicheverZoneExists(t *testing.T) { + // Real-world shape: today (13th) is published under zone SU, yesterday (12th) + // under RE, tomorrow (14th) not yet published. The provider must find both + // and never care which zone a given day used. + client := fakeCDN(map[string]canned{ + "20260713/SU": {etag: "e13su", body: "%PDF-13"}, + "20260712/RE": {etag: "e12re", body: "%PDF-12"}, + }) + wp := New("", nil, "", "") + + eds, versions, err := wp.Poll(context.Background(), source.Deps{HTTP: client}, nil, now) + if err != nil { + t.Fatalf("Poll: %v", err) + } + if len(eds) != 2 { + t.Fatalf("got %d editions, want 2", len(eds)) + } + + byDate := map[string]source.Edition{} + for _, e := range eds { + if e.Media != source.MediaPDF { + t.Errorf("edition media = %q, want PDF", e.Media) + } + byDate[e.Date.Format("20060102")] = e + } + if e, ok := byDate["20260713"]; !ok || string(e.Data) != "%PDF-13" || e.Version != "e13su" { + t.Errorf("20260713 edition = %+v, want body %%PDF-13 / etag e13su", e) + } + if e, ok := byDate["20260712"]; !ok || string(e.Data) != "%PDF-12" || e.Version != "e12re" { + t.Errorf("20260712 edition = %+v, want body %%PDF-12 / etag e12re", e) + } + // Only the two 200 URLs carry tokens; every 403 probe is dropped. + if len(versions) != 2 { + t.Errorf("versions = %v, want exactly the 2 present editions", versions) + } +} + +func TestPoll_WarmIsConditionalAndReturnsNothing(t *testing.T) { + client := fakeCDN(map[string]canned{ + "20260713/SU": {etag: "e13su", body: "%PDF-13"}, + "20260712/RE": {etag: "e12re", body: "%PDF-12"}, + }) + wp := New("", nil, "", "") + + seen := map[string]string{ + wp.url(now, "SU"): "e13su", + wp.url(now.AddDate(0, 0, -1), "RE"): "e12re", + } + eds, versions, err := wp.Poll(context.Background(), source.Deps{HTTP: client}, seen, now) + if err != nil { + t.Fatalf("Poll: %v", err) + } + if len(eds) != 0 { + t.Fatalf("got %d editions, want 0 (all 304)", len(eds)) + } + if versions[wp.url(now, "SU")] != "e13su" { + t.Errorf("today's SU version = %q, want retained e13su", versions[wp.url(now, "SU")]) + } +} + +func TestPoll_ForbiddenIsAbsentNotError(t *testing.T) { + // Nothing published for any probed day/zone: 403s everywhere. That is a + // healthy (if empty) poll, not a failure, and any stale token is dropped so + // the next poll re-probes cleanly. + client := fakeCDN(map[string]canned{}) + wp := New("", nil, "", "") + + seen := map[string]string{wp.url(now, "SU"): "e-old"} + eds, versions, err := wp.Poll(context.Background(), source.Deps{HTTP: client}, seen, now) + if err != nil { + t.Fatalf("Poll: %v (403 is absent, not an error)", err) + } + if len(eds) != 0 { + t.Fatalf("got %d editions, want 0", len(eds)) + } + if _, ok := versions[wp.url(now, "SU")]; ok { + t.Errorf("absent URL kept a token %q; want it dropped", versions[wp.url(now, "SU")]) + } +} + +func TestPoll_NonPDFBodyIsProbeErrorAndRetainsToken(t *testing.T) { + // A 200 whose body isn't a PDF must not become an edition, and the old token + // must be retained so the next poll retries instead of 304ing past the real + // file. + client := fakeCDN(map[string]canned{ + "20260713/SU": {etag: "e-bad", body: "oops"}, + }) + wp := New("", nil, "", "") + + seen := map[string]string{wp.url(now, "SU"): "e-old"} + eds, versions, err := wp.Poll(context.Background(), source.Deps{HTTP: client}, seen, now) + if err != nil { + t.Fatalf("Poll: %v (one bad probe must not sink the poll)", err) + } + if len(eds) != 0 { + t.Fatalf("got %d editions, want 0 (non-PDF body rejected)", len(eds)) + } + if versions[wp.url(now, "SU")] != "e-old" { + t.Errorf("SU version = %q, want retained e-old (not burned e-bad)", versions[wp.url(now, "SU")]) + } +} + +func TestPoll_EditionDateComesFromFolderNotClock(t *testing.T) { + // The edition date is the folder date encoded in the URL, exact and never + // zero — regardless of any Last-Modified header (the provider ignores it). + client := fakeCDN(map[string]canned{ + "20260712/SU": {etag: "e12", body: "%PDF-12"}, + }) + wp := New("", nil, "", "") + + eds, _, err := wp.Poll(context.Background(), source.Deps{HTTP: client}, nil, now) + if err != nil { + t.Fatalf("Poll: %v", err) + } + if len(eds) != 1 { + t.Fatalf("got %d editions, want 1", len(eds)) + } + if eds[0].Date.IsZero() { + t.Fatal("edition date is zero; the archive rejects zero dates") + } + if got := eds[0].Date.Format("20060102"); got != "20260712" { + t.Errorf("edition date = %s, want 20260712 (the folder date)", got) + } +} + +func TestPoll_AllProbesFailIsError(t *testing.T) { + // Every probe failing at the transport level (upstream unreachable) is the + // one condition that surfaces as an error. + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("dial tcp: connection refused") + })} + wp := New("", nil, "", "") + + _, _, err := wp.Poll(context.Background(), source.Deps{HTTP: client}, nil, now) + if err == nil { + t.Fatal("want an error when every probe fails at the transport level") + } +} + +func TestPoll_AllUnexpectedStatusIsError(t *testing.T) { + // A total upstream outage that still speaks HTTP (every probe 5xx) must + // surface as a failure, not a silent healthy no-op — otherwise a prolonged + // outage looks like a run of healthy polls in the health record. + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusInternalServerError, Header: http.Header{}, Request: r, + Body: io.NopCloser(strings.NewReader("boom")), + }, nil + })} + wp := New("", nil, "", "") + + _, _, err := wp.Poll(context.Background(), source.Deps{HTTP: client}, nil, now) + if err == nil { + t.Fatal("want an error when every probe returns an unexpected status") + } +} + +func TestPoll_OneUnexpectedStatusAmongHitsIsHealthy(t *testing.T) { + // A single odd status among otherwise-determinate probes is not a poll + // failure: only an all-failed poll errors. Today's SU serves a real PDF; + // every other probe 5xx's. + wp := New("", nil, "", "") + todaySU := wp.url(now, "SU") + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + if r.URL.String() == todaySU { + h := http.Header{} + h.Set("ETag", "e13") + return &http.Response{StatusCode: http.StatusOK, Header: h, Request: r, + Body: io.NopCloser(strings.NewReader("%PDF-13"))}, nil + } + return &http.Response{StatusCode: http.StatusBadGateway, Header: http.Header{}, Request: r, + Body: io.NopCloser(strings.NewReader("bad gateway"))}, nil + })} + + eds, _, err := wp.Poll(context.Background(), source.Deps{HTTP: client}, nil, now) + if err != nil { + t.Fatalf("Poll: %v (a partial success must not error)", err) + } + if len(eds) != 1 { + t.Fatalf("got %d editions, want 1 (the one 200 among 5xx)", len(eds)) + } +} + +func TestPoll_PDFHeaderWithinFirstKilobyteIsAccepted(t *testing.T) { + client := fakeCDN(map[string]canned{ + "20260713/SU": {etag: "e13", body: "\xef\xbb\xbf junk %PDF-1.4 rest"}, + }) + wp := New("", nil, "", "") + + eds, _, err := wp.Poll(context.Background(), source.Deps{HTTP: client}, nil, now) + if err != nil { + t.Fatalf("Poll: %v", err) + } + if len(eds) != 1 { + t.Fatalf("got %d editions, want 1 (header within first 1KB accepted)", len(eds)) + } +} + +func TestNew_DefaultsAndOverrides(t *testing.T) { + d := New("", nil, "", "") + if d.Page != "A01" || d.Edition != "EZ" || d.Product != "DAILY" { + t.Errorf("defaults = %+v, want A01/EZ/DAILY", d) + } + if len(d.Zones) != 2 || d.Zones[0] != "SU" || d.Zones[1] != "RE" { + t.Errorf("default zones = %v, want [SU RE]", d.Zones) + } + o := New("Z09", []string{"DC"}, "MD", "SUNDAY") + if o.Page != "Z09" || o.Edition != "MD" || o.Product != "SUNDAY" || len(o.Zones) != 1 || o.Zones[0] != "DC" { + t.Errorf("overrides not honored: %+v", o) + } + if got := o.url(now, "DC"); got != baseURL+"/20260713/Z09_DC_MD_SUNDAY_20260713.pdf" { + t.Errorf("url = %s", got) + } +} diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index 940e514..a9c6fb6 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -259,6 +259,10 @@ func (r *Reconciler) ReconcileSource(ctx context.Context, src source.Source, now } if stored > 0 { + // Stamp the archive with the source's display name so its history stays + // labeled after the paper leaves the catalog (the archive is self- + // describing; the store row may be pruned). + _ = r.Archive.SetName(src.ID, src.DisplayName) _ = r.Store.RecordSuccess(src.ID, now) log.Info("archived editions", "source", src.ID, "count", stored) } diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index 5ddbc6d..d69c08f 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -52,7 +52,7 @@ func TestReconcileOnce_ArchivesAndPersists(t *testing.T) { }}, versions: map[string]string{"url30": "e30"}, } - src := source.Source{ID: "ny-nyt", Provider: prov} + src := source.Source{ID: "ny-nyt", DisplayName: "The New York Times", Provider: prov} r, arch, store := newReconciler(t, []source.Source{src}) r.ReconcileOnce(context.Background()) @@ -66,6 +66,11 @@ func TestReconcileOnce_ArchivesAndPersists(t *testing.T) { if rec := store.Snapshot().Sources["ny-nyt"]; rec.LastFetchOK == nil { t.Error("expected LastFetchOK recorded after storing an edition") } + // The archive is stamped with the display name so its history stays labeled + // after the paper leaves the catalog. + if got := arch.Name("ny-nyt"); got != "The New York Times" { + t.Errorf("archive label = %q, want the source's display name", got) + } } func TestReconcileOnce_PollErrorRecordsFailure(t *testing.T) { diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 2f232b8..b99fcc3 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -11,6 +11,7 @@ import ( "github.com/kelchm/broadsheet/internal/catalog" "github.com/kelchm/broadsheet/internal/provider/freedomforum" + "github.com/kelchm/broadsheet/internal/provider/washingtonpost" "github.com/kelchm/broadsheet/internal/source" ) @@ -32,6 +33,16 @@ func Decode(providerType string, config json.RawMessage) (source.Provider, error return nil, fmt.Errorf("registry: freedomforum config needs a prefix") } return freedomforum.FreedomForum{Prefix: c.Prefix}, nil + case "washingtonpost": + // Every field is optional — the provider defaults the front-page page, + // zone candidates, edition, and product. An empty config is valid. + var w washingtonpost.WashingtonPost + if len(config) > 0 { + if err := json.Unmarshal(config, &w); err != nil { + return nil, fmt.Errorf("registry: washingtonpost config: %w", err) + } + } + return washingtonpost.New(w.Page, w.Zones, w.Edition, w.Product), nil default: return nil, fmt.Errorf("registry: unknown provider type %q", providerType) } diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go index 9c700f9..e0fa8a9 100644 --- a/internal/registry/registry_test.go +++ b/internal/registry/registry_test.go @@ -46,6 +46,25 @@ func TestDecode(t *testing.T) { } } +func TestDecodeWashingtonPost(t *testing.T) { + // An empty config is valid — every field defaults. + if p, err := Decode("washingtonpost", []byte(`{}`)); err != nil || p == nil { + t.Fatalf("Decode washingtonpost {}: %v", err) + } + // A nil/absent config is valid too. + if p, err := Decode("washingtonpost", nil); err != nil || p == nil { + t.Fatalf("Decode washingtonpost nil: %v", err) + } + // Overrides decode. + if p, err := Decode("washingtonpost", []byte(`{"zones":["DC"],"product":"SUNDAY"}`)); err != nil || p == nil { + t.Fatalf("Decode washingtonpost override: %v", err) + } + // Malformed JSON errors. + if _, err := Decode("washingtonpost", []byte(`not json`)); err == nil { + t.Error("bad JSON must error") + } +} + func TestEveryCatalogEntryDecodes(t *testing.T) { // The catalog ships enabled-able data; a typo'd provider type or empty // prefix must fail at build time, not vanish a paper at runtime. diff --git a/internal/store/store.go b/internal/store/store.go index 1e1ff6d..d1ef8a9 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -137,27 +137,108 @@ type SourceRow struct { Position int } -// SeedSources inserts rows that don't already exist (by ID). Existing rows are -// left untouched — user edits win over catalog updates. -func (s *Store) SeedSources(rows []SourceRow) error { +// SeedSources reconciles the store against the catalog. The catalog owns a +// paper's identity and wiring (display name, provider type + config, catalog +// position); the store owns the user's one choice, its enabled flag. Every boot +// it fully reconciles: a new paper is inserted with its catalog default; an +// existing paper's wiring is refreshed from the catalog (that is how a catalog +// fix — a repointed provider, a corrected config, a rename — reaches installs +// that already have the row) while the user's enabled toggle is never clobbered +// by the catalog's default; and a paper the catalog has dropped is deleted, so a +// removed or permanently-broken source disappears from existing installs, not +// only from fresh ones. (The Enabled field carried in each row is only the +// catalog default; SetSourceEnabled is the sole writer of a user's choice.) +// +// Deletion is safe because the sources table is wholly catalog-derived — the +// only writer of a row is this method, and embedders that supply their own +// sources bypass the store entirely (Config.Sources) — so an id absent from the +// catalog is unambiguously a removed paper. The one caveat is a downgrade: +// running an older binary whose embedded catalog predates papers a newer binary +// added would prune those rows (they return, at their catalog default, on the +// next upgrade). An empty catalog never prunes, so a caller passing nothing +// can't wipe the table. +// +// When prune is false the drop step is skipped (upsert only). The caller uses +// this to avoid deleting a dropped paper's row before its archived display name +// was safely preserved — losing the name is worse than a stale row that the next +// (pruning) boot cleans up. +func (s *Store) SeedSources(rows []SourceRow, prune bool) error { + if len(rows) == 0 { + return nil + } tx, err := s.db.Begin() if err != nil { return fmt.Errorf("store: seed sources: %w", err) } + + // Snapshot existing ids up front so we can prune the ones the catalog no + // longer carries. Read fully and close before issuing writes on the tx. + existing := map[string]bool{} + idRows, err := tx.Query(`SELECT id FROM sources`) + if err != nil { + _ = tx.Rollback() + return fmt.Errorf("store: seed sources: read ids: %w", err) + } + for idRows.Next() { + var id string + if err := idRows.Scan(&id); err != nil { + _ = idRows.Close() + _ = tx.Rollback() + return fmt.Errorf("store: seed sources: scan id: %w", err) + } + existing[id] = true + } + if err := idRows.Err(); err != nil { + _ = idRows.Close() + _ = tx.Rollback() + return fmt.Errorf("store: seed sources: read ids: %w", err) + } + _ = idRows.Close() + + incoming := make(map[string]bool, len(rows)) for _, r := range rows { + incoming[r.ID] = true cfg := r.ProviderConfig if len(cfg) == 0 { cfg = json.RawMessage("{}") } if _, err := tx.Exec( - `INSERT OR IGNORE INTO sources (id, display_name, provider_type, provider_config, enabled, position) - VALUES (?, ?, ?, ?, ?, ?)`, + `INSERT INTO sources (id, display_name, provider_type, provider_config, enabled, position) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + display_name = excluded.display_name, + provider_type = excluded.provider_type, + provider_config = excluded.provider_config, + position = excluded.position`, r.ID, r.DisplayName, r.ProviderType, string(cfg), r.Enabled, r.Position, ); err != nil { _ = tx.Rollback() return fmt.Errorf("store: seed source %s: %w", r.ID, err) } } + + // Prune papers the catalog has dropped, plus their dependent state, so a + // removed source leaves nothing behind (a stale row would keep the reconciler + // polling a dead feed forever and clutter the catalog UI). Skipped when the + // caller couldn't first preserve dropped papers' archived names. + if prune { + for id := range existing { + if incoming[id] { + continue + } + for _, stmt := range []string{ + `DELETE FROM sources WHERE id = ?`, + `DELETE FROM provider_versions WHERE source_id = ?`, + `DELETE FROM fetch_events WHERE source_id = ?`, + `DELETE FROM crop_overrides WHERE source_id = ?`, + } { + if _, err := tx.Exec(stmt, id); err != nil { + _ = tx.Rollback() + return fmt.Errorf("store: prune source %s: %w", id, err) + } + } + } + } return tx.Commit() } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 916e072..f7cc598 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -38,40 +38,157 @@ func TestOpen_MigratesAndReopens(t *testing.T) { } } -func TestSeedSources_ExistingRowsWin(t *testing.T) { +func TestSeedSources_RefreshesWiringButPreservesUserEnabled(t *testing.T) { s := open(t) rows := []SourceRow{ {ID: "a", DisplayName: "Paper A", ProviderType: "freedomforum", ProviderConfig: json.RawMessage(`{"prefix":"A_A"}`), Enabled: true, Position: 1}, - {ID: "b", DisplayName: "Paper B", ProviderType: "freedomforum", Enabled: false, Position: 2}, + {ID: "b", DisplayName: "Paper B", ProviderType: "freedomforum", + ProviderConfig: json.RawMessage(`{"prefix":"B_B"}`), Enabled: false, Position: 2}, } - if err := s.SeedSources(rows); err != nil { + if err := s.SeedSources(rows, true); err != nil { t.Fatalf("SeedSources: %v", err) } - // Re-seeding with different values must NOT clobber user state. + // The user enables b (b's catalog default is disabled) — their one choice. + if err := s.SetSourceEnabled("b", true); err != nil { + t.Fatalf("SetSourceEnabled: %v", err) + } + + // A later release reconciles the catalog: a is repointed to a new provider + // and renamed, and b's catalog default is still disabled. Re-seeding must + // refresh the wiring (so a catalog fix reaches this existing install) but + // never override the user's enabled toggle with the catalog default. if err := s.SeedSources([]SourceRow{ - {ID: "a", DisplayName: "RENAMED", ProviderType: "freedomforum", Enabled: false}, - }); err != nil { + {ID: "a", DisplayName: "Paper A Renamed", ProviderType: "washingtonpost", + ProviderConfig: json.RawMessage(`{}`), Enabled: false, Position: 1}, + {ID: "b", DisplayName: "Paper B", ProviderType: "freedomforum", + ProviderConfig: json.RawMessage(`{"prefix":"B_B"}`), Enabled: false, Position: 2}, + {ID: "c", DisplayName: "New Paper C", ProviderType: "freedomforum", + ProviderConfig: json.RawMessage(`{"prefix":"C_C"}`), Enabled: true, Position: 3}, + }, true); err != nil { t.Fatalf("re-seed: %v", err) } + byID := map[string]SourceRow{} + all, err := s.ListSources(false) + if err != nil { + t.Fatalf("ListSources: %v", err) + } + for _, r := range all { + byID[r.ID] = r + } + if len(all) != 3 { + t.Fatalf("got %d sources, want 3 (c newly seeded)", len(all)) + } + + // a: wiring refreshed from the catalog... + if got := byID["a"]; got.DisplayName != "Paper A Renamed" || + got.ProviderType != "washingtonpost" || string(got.ProviderConfig) != "{}" { + t.Errorf("row a = %+v, want wiring refreshed (renamed, repointed to washingtonpost)", byID["a"]) + } + // ...but a's user-enabled state (true) survives the catalog default of false. + if !byID["a"].Enabled { + t.Error("row a enabled = false, want the user's enabled=true preserved over the catalog default") + } + // b: user enabled it; the catalog default (disabled) must not flip it back. + if !byID["b"].Enabled { + t.Error("row b enabled = false, want the user's enable to survive re-seed") + } + // c: a genuinely new catalog paper appears, with its catalog default enabled. + if !byID["c"].Enabled || byID["c"].ProviderType != "freedomforum" { + t.Errorf("row c = %+v, want newly inserted and enabled by its catalog default", byID["c"]) + } +} + +func TestSeedSources_PrunesDroppedPapers(t *testing.T) { + s := open(t) + seed := func(ids ...string) { + rows := make([]SourceRow, len(ids)) + for i, id := range ids { + rows[i] = SourceRow{ID: id, DisplayName: id, ProviderType: "freedomforum", + ProviderConfig: json.RawMessage(`{"prefix":"X"}`), Position: i} + } + if err := s.SeedSources(rows, true); err != nil { + t.Fatalf("SeedSources(%v): %v", ids, err) + } + } + seed("a", "b", "c") + // Give c some dependent state (version token + a health event). + if err := s.SetVersions("c", map[string]string{"http://x": "etag-c"}); err != nil { + t.Fatal(err) + } + if err := s.RecordFailure("c", "boom", time.Unix(1000, 0)); err != nil { + t.Fatal(err) + } + + // The catalog drops c. + seed("a", "b") + all, err := s.ListSources(false) if err != nil { t.Fatalf("ListSources: %v", err) } if len(all) != 2 { - t.Fatalf("got %d sources, want 2", len(all)) + t.Fatalf("got %d sources, want 2 (c pruned)", len(all)) } - if all[0].ID != "a" || all[0].DisplayName != "Paper A" || !all[0].Enabled { - t.Errorf("row a = %+v, want original values preserved (seed must not clobber)", all[0]) + for _, r := range all { + if r.ID == "c" { + t.Fatal("c still present after being dropped from the catalog") + } } - if string(all[1].ProviderConfig) != "{}" { - t.Errorf("empty config = %q, want {}", all[1].ProviderConfig) + // Dependent state for c must be gone too. + if v := s.Versions("c"); len(v) != 0 { + t.Errorf("pruned source c still has versions %v", v) } + health, err := s.HealthSnapshot() + if err != nil { + t.Fatalf("HealthSnapshot: %v", err) + } + if _, ok := health["c"]; ok { + t.Error("pruned source c still has health history") + } +} - enabled, err := s.ListSources(true) - if err != nil || len(enabled) != 1 || enabled[0].ID != "a" { - t.Errorf("enabled set = %+v (err %v), want just a", enabled, err) +func TestSeedSources_EmptyCatalogNeverPrunes(t *testing.T) { + s := open(t) + if err := s.SeedSources([]SourceRow{ + {ID: "a", DisplayName: "A", ProviderType: "freedomforum", ProviderConfig: json.RawMessage(`{"prefix":"A"}`)}, + }, true); err != nil { + t.Fatalf("SeedSources: %v", err) + } + // A nil/empty seed is a no-op guard, not a table wipe. + if err := s.SeedSources(nil, true); err != nil { + t.Fatalf("SeedSources(nil, true): %v", err) + } + if n, err := s.CountSources(); err != nil || n != 1 { + t.Fatalf("CountSources = %d, %v; want the row preserved", n, err) + } +} + +func TestSeedSources_NoPruneKeepsDroppedRows(t *testing.T) { + s := open(t) + seed := func(prune bool, ids ...string) { + rows := make([]SourceRow, len(ids)) + for i, id := range ids { + rows[i] = SourceRow{ID: id, DisplayName: id, ProviderType: "freedomforum", + ProviderConfig: json.RawMessage(`{"prefix":"X"}`), Position: i} + } + if err := s.SeedSources(rows, prune); err != nil { + t.Fatalf("SeedSources(%v, prune=%v): %v", ids, prune, err) + } + } + seed(true, "a", "b") + // prune=false: "b" left the catalog, but its row must survive (its archived + // name may not have been preserved yet — losing it is worse than a stale row). + seed(false, "a") + if all, _ := s.ListSources(false); len(all) != 2 { + t.Fatalf("got %d sources, want 2 (b retained under prune=false)", len(all)) + } + // A later reconcile with prune=true actually drops it. + seed(true, "a") + all, _ := s.ListSources(false) + if len(all) != 1 || all[0].ID != "a" { + t.Fatalf("got %v, want just [a] after a pruning reconcile", all) } } diff --git a/pkg/broadsheet/broadsheet.go b/pkg/broadsheet/broadsheet.go index 249016f..d6c2f63 100644 --- a/pkg/broadsheet/broadsheet.go +++ b/pkg/broadsheet/broadsheet.go @@ -316,16 +316,31 @@ func New(cfg Config) (*Engine, error) { } importLegacyState(st, filepath.Join(cfg.DataDir, "state.json"), cfg.Logger) + // Build the archive before reconciling sources: loadSources stamps archive + // labels before it prunes, so a paper dropped in this release keeps its + // history labeled (and the archive stays self-describing). + arch := &archive.Store{Root: archiveDir} + srcs := cfg.Sources if srcs == nil { - srcs, err = loadSources(st, cfg.Logger) + srcs, err = loadSources(st, arch, cfg.Logger) if err != nil { _ = st.Close() return nil, err } + } else { + // Embedder mode has no store to reconcile, but its archives should still + // be self-describing — label them from the configured sources. There's no + // prune here, so a label write failure is non-fatal; just note it. + names := make(map[string]string, len(srcs)) + for _, s := range srcs { + names[s.ID] = s.DisplayName + } + if err := stampArchiveLabels(arch, names); err != nil && cfg.Logger != nil { + cfg.Logger.Warn("archive label stamping failed", "err", err) + } } - arch := &archive.Store{Root: archiveDir} p := &Engine{ cfg: cfg, sources: srcs, @@ -354,6 +369,32 @@ func New(cfg Config) (*Engine, error) { return p, nil } +// stampArchiveLabels writes each source's display name into its archive metadata +// so history stays labeled — and the archive stays self-describing and portable — +// after a paper leaves the catalog and its store row is pruned. It only touches +// ids that already have an archive directory (SetName is a no-op for a blank name +// and rewrites only when the name changed), so it never creates directories and +// never clobbers a transplanted archive that already carries its own label. +// stampArchiveLabels writes each source's display name into its archive metadata +// so history stays labeled — and the archive stays self-describing and portable — +// after a paper leaves the catalog and its store row is pruned. It only touches +// ids that already have an archive directory (SetName never creates one), and +// SetName rewrites only when the name changed. An id absent from names (a foreign +// paper transplanted into the archive, not in the catalog or store) is skipped, +// so its own label is left untouched; a known id is (re)labeled from names, which +// is the intended refresh. It returns the first write error so the caller can +// avoid a destructive prune when a soon-to-be-dropped paper's name wasn't saved. +func stampArchiveLabels(arch *archive.Store, names map[string]string) error { + for _, id := range arch.SourceIDs() { + if name := names[id]; name != "" { + if err := arch.SetName(id, name); err != nil { + return fmt.Errorf("broadsheet: stamp archive label %q: %w", id, err) + } + } + } + return nil +} + // getSources returns the active source set (an immutable snapshot). func (p *Engine) getSources() []source.Source { p.srcMu.RLock() @@ -432,9 +473,22 @@ func (p *Engine) Catalog() ([]CatalogEntry, error) { return out, nil } +// ArchiveName returns the display name captured alongside a source's archive +// while it was archiving, or "" if none. It lets the archive browser label a +// paper that has left the catalog — whose live catalog name is gone — with the +// name its history was collected under, instead of a bare id. The archive is +// self-describing: identity travels with the data, not the catalog row. +func (p *Engine) ArchiveName(id string) string { + return p.archive.Name(id) +} + // ArchiveIndex returns edition dates (oldest first, same-day duplicates -// collapsed) for every source holding at least one archived edition — -// including disabled papers, whose history remains browsable. +// collapsed) for every source holding at least one archived edition — including +// disabled papers and papers dropped from the catalog entirely, whose collected +// history stays browsable until it ages out on the normal retention. Catalog +// membership governs polling and the catalog UI; the archive is independent data +// with its own age-based lifetime, so a dropped paper's front pages remain here +// (and renderable, via knownSource) rather than vanishing the moment it's removed. func (p *Engine) ArchiveIndex() map[string][]time.Time { out := map[string][]time.Time{} for _, id := range p.archive.SourceIDs() { @@ -486,16 +540,47 @@ func (p *Engine) RenderEdition(ctx context.Context, sourceID string, date time.T } // loadSources seeds the store from the embedded catalog, then loads the -// enabled set. Seeding runs every boot: INSERT OR IGNORE means papers added to -// the catalog in a new release appear in the store, while rows the user has -// touched are never clobbered. Sources live as rows — provider type + JSON -// config — decoded into typed providers here; a row that fails to decode is +// enabled set. Seeding runs every boot and fully reconciles the store to the +// catalog (see store.SeedSources): papers added to the catalog appear, dropped +// papers are pruned, and existing papers' wiring is refreshed — all while the +// user's enabled toggles are preserved. Sources live as rows — provider type + +// JSON config — decoded into typed providers here; a row that fails to decode is // skipped with a warning rather than taking the whole engine down. -func loadSources(st *store.Store, logger *slog.Logger) ([]source.Source, error) { +func loadSources(st *store.Store, arch *archive.Store, logger *slog.Logger) ([]source.Source, error) { entries, err := catalog.All() if err != nil { return nil, fmt.Errorf("broadsheet: load catalog: %w", err) } + + // Stamp archive labels BEFORE reconciling. Names come from the current store + // rows (which still hold the name of a paper about to be pruned) overlaid with + // the catalog (fresh names win for papers that survive). Doing this ahead of + // SeedSources' prune is what lets a paper dropped in *this* release keep its + // history labeled; disabled papers, which never re-archive, are covered the + // same way. The reconciler keeps active papers' labels fresh on each Put. + names := map[string]string{} + stored, err := st.ListSources(false) + if err != nil { + return nil, fmt.Errorf("broadsheet: list sources for archive labeling: %w", err) + } + for _, r := range stored { + names[r.ID] = r.DisplayName + } + for _, e := range entries { + names[e.ID] = e.Name + } + // If a label write fails, a paper about to be dropped this boot might lose the + // name it needs to keep its archived history readable — so skip the prune this + // boot rather than delete a row whose name we couldn't preserve. The upsert + // still runs; the prune retries on the next boot once the write succeeds. + prune := true + if err := stampArchiveLabels(arch, names); err != nil { + if logger != nil { + logger.Warn("archive label stamping failed; skipping catalog prune this boot", "err", err) + } + prune = false + } + rows := make([]store.SourceRow, 0, len(entries)) for i, e := range entries { rows = append(rows, store.SourceRow{ @@ -504,7 +589,7 @@ func loadSources(st *store.Store, logger *slog.Logger) ([]source.Source, error) Enabled: e.Default, Position: i, }) } - if err := st.SeedSources(rows); err != nil { + if err := st.SeedSources(rows, prune); err != nil { return nil, fmt.Errorf("broadsheet: seed sources: %w", err) } return loadEnabled(st, logger) @@ -533,14 +618,21 @@ func loadEnabled(st *store.Store, logger *slog.Logger) ([]source.Source, error) return out, nil } -// knownSource reports whether an ID exists at all — in the live set, or (for -// store-backed engines) anywhere in the catalog. Editions/refresh endpoints -// must address disabled papers too: the API advertises the full catalog, and -// the archive keeps serving a paper's history after it's toggled off. +// knownSource reports whether an ID is addressable for reads — in the live set, +// anywhere in the catalog (for store-backed engines), or holding archived +// editions on disk. Editions/refresh endpoints must address disabled papers too +// (the API advertises the full catalog), and archived history outlives catalog +// membership: a paper toggled off — or dropped from the catalog entirely — stays +// readable and renderable until its editions age out on the normal retention. +// Catalog membership and the archive are independent lifetimes. func (p *Engine) knownSource(id string) bool { if source.ByID(p.getSources(), id) != nil { return true } + // Archived history keeps a paper addressable even with no catalog row left. + if _, ok := p.archive.Newest(id); ok { + return true + } if p.cfg.Sources != nil { return false } diff --git a/pkg/broadsheet/broadsheet_test.go b/pkg/broadsheet/broadsheet_test.go index b7647a2..81b8aac 100644 --- a/pkg/broadsheet/broadsheet_test.go +++ b/pkg/broadsheet/broadsheet_test.go @@ -3,6 +3,7 @@ package broadsheet import ( "bytes" "context" + "encoding/json" "image" "image/color" "os" @@ -16,6 +17,7 @@ import ( "github.com/kelchm/broadsheet/internal/archive" "github.com/kelchm/broadsheet/internal/catalog" "github.com/kelchm/broadsheet/internal/source" + "github.com/kelchm/broadsheet/internal/store" ) func TestMemCursors_AdvancePerDevice(t *testing.T) { @@ -135,10 +137,10 @@ func TestServe_AppliesCrop(t *testing.T) { } } -// uniformPNG returns PNG bytes for a w x h image of the given gray level. -func uniformPNG(t *testing.T, w, h int, level uint8) []byte { +// uniformPNG returns PNG bytes for a small fixture image of the given gray level. +func uniformPNG(t *testing.T, level uint8) []byte { t.Helper() - img := imaging.New(w, h, color.NRGBA{R: level, G: level, B: level, A: 255}) + img := imaging.New(32, 48, color.NRGBA{R: level, G: level, B: level, A: 255}) var buf bytes.Buffer if err := imaging.Encode(&buf, img, imaging.PNG); err != nil { t.Fatalf("encode fixture png: %v", err) @@ -148,14 +150,15 @@ func uniformPNG(t *testing.T, w, h int, level uint8) []byte { // newTestEngine builds an engine over a temp DataDir with one MediaImage // edition archived for source "a". Providers are nil: these tests never poll. -func newTestEngine(t *testing.T, width int, srcIDs ...string) (*Engine, *archive.Store, time.Time) { +func newTestEngine(t *testing.T, srcIDs ...string) (*Engine, *archive.Store, time.Time) { t.Helper() + const width = 64 dir := t.TempDir() date := time.Date(2026, 6, 30, 0, 0, 0, 0, time.UTC) arch := &archive.Store{Root: filepath.Join(dir, "archive")} if _, err := arch.Put("a", source.Edition{ - Date: date, Media: source.MediaImage, Data: uniformPNG(t, 32, 48, 0), // black + Date: date, Media: source.MediaImage, Data: uniformPNG(t, 0), // black }); err != nil { t.Fatalf("archive.Put: %v", err) } @@ -171,8 +174,140 @@ func newTestEngine(t *testing.T, width int, srcIDs ...string) (*Engine, *archive return p, arch, date } +func TestKnownSource_ArchivedHistoryOutlivesCatalog(t *testing.T) { + // No configured sources, but "a" has an archived edition — the shape of a + // paper dropped from the catalog whose archive hasn't yet aged out. Catalog + // membership and archive retention are independent, so "a" stays addressable + // (listable + renderable); an id with neither a config row nor an archive is + // genuinely unknown. + p, arch, date := newTestEngine(t) // archives "a", configures nothing + // "a" archived under its real name while it was active — the archive is + // self-describing, so its identity survives leaving the catalog. + if err := arch.SetName("a", "The A Paper"); err != nil { + t.Fatalf("SetName: %v", err) + } + + if !p.knownSource("a") { + t.Fatal("archived-but-unconfigured source should stay known") + } + eds, err := p.ListEditions("a") + if err != nil || len(eds) != 1 || !eds[0].Equal(date) { + t.Fatalf("ListEditions(a) = %v, %v; want the one archived date", eds, err) + } + if _, err := p.RenderEdition(context.Background(), "a", date); err != nil { + t.Fatalf("RenderEdition(a) must render archived history: %v", err) + } + // Its history is labeled with the real name, not a bare id — sourced from the + // archive itself, since no catalog row remains. + if got := p.ArchiveName("a"); got != "The A Paper" { + t.Errorf("ArchiveName(a) = %q, want the archived label 'The A Paper'", got) + } + if p.knownSource("z") { + t.Error("an id with no config row and no archive must be unknown") + } +} + +func TestBackfillArchiveLabels_LabelsExistingArchivesAtStartup(t *testing.T) { + // An existing deploy: a catalog paper ("ny-nyt") already has an archive + // written before self-describing metadata existed — no label. Booting the + // store-backed engine must backfill the label from the catalog name, so the + // paper's history keeps its name if it's later dropped (and disabled papers, + // which never re-archive, are covered the same way). + dir := t.TempDir() + arch := &archive.Store{Root: filepath.Join(dir, "archive")} + if _, err := arch.Put("ny-nyt", source.Edition{ + Date: time.Date(2026, 6, 30, 0, 0, 0, 0, time.UTC), + Media: source.MediaImage, Data: uniformPNG(t, 0), + }); err != nil { + t.Fatalf("archive.Put: %v", err) + } + if got := arch.Name("ny-nyt"); got != "" { + t.Fatalf("precondition: archive should be unlabeled, got %q", got) + } + + p, err := New(Config{DataDir: dir}) // store-backed: seeds the embedded catalog + if err != nil { + t.Fatalf("New: %v", err) + } + if got := p.ArchiveName("ny-nyt"); got != "The New York Times" { + t.Errorf("ArchiveName after backfill = %q, want the catalog name", got) + } +} + +func TestLoadSources_LabelsDroppedPaperBeforePruning(t *testing.T) { + // The ordering guarantee: a paper present in an existing install's store but + // NOT in the current catalog (dropped in this release), with an unlabeled + // archive, must have its name stamped onto the archive BEFORE the reconcile + // prune deletes its row — so its history keeps its real name, not a bare id. + dir := t.TempDir() + st, err := store.Open(filepath.Join(dir, "broadsheet.db")) + if err != nil { + t.Fatalf("store.Open: %v", err) + } + if err := st.SeedSources([]store.SourceRow{{ + ID: "gone-paper", DisplayName: "The Gone Gazette", + ProviderType: "freedomforum", ProviderConfig: json.RawMessage(`{"prefix":"X"}`), + }}, true); err != nil { + t.Fatalf("seed pre-existing row: %v", err) + } + _ = st.Close() + + arch := &archive.Store{Root: filepath.Join(dir, "archive")} + if _, err := arch.Put("gone-paper", source.Edition{ + Date: time.Date(2026, 6, 30, 0, 0, 0, 0, time.UTC), + Media: source.MediaImage, Data: uniformPNG(t, 0), + }); err != nil { + t.Fatalf("archive.Put: %v", err) + } + + // New() reconciles the embedded catalog (which has no "gone-paper"), so its + // row is pruned — but its archive was labeled first. + p, err := New(Config{DataDir: dir}) + if err != nil { + t.Fatalf("New: %v", err) + } + if got := p.ArchiveName("gone-paper"); got != "The Gone Gazette" { + t.Errorf("ArchiveName = %q, want the name preserved before the prune", got) + } + if !p.knownSource("gone-paper") { + t.Error("dropped-but-archived paper should still be addressable via its archive") + } +} + +func TestArchive_PortableDropIn(t *testing.T) { + // The portability property: a self-describing archive directory transplanted + // into an install — an id in no catalog and no store — is browsable, + // renderable, and shows its real name from its own metadata. Startup labeling + // must NOT clobber the transplanted label with a blank. + dir := t.TempDir() + arch := &archive.Store{Root: filepath.Join(dir, "archive")} + if _, err := arch.Put("foreign", source.Edition{ + Date: time.Date(2026, 6, 30, 0, 0, 0, 0, time.UTC), + Media: source.MediaImage, Data: uniformPNG(t, 0), + }); err != nil { + t.Fatalf("archive.Put: %v", err) + } + if err := arch.SetName("foreign", "Le Journal"); err != nil { + t.Fatalf("SetName: %v", err) + } + + p, err := New(Config{DataDir: dir}) // seeds the embedded catalog; "foreign" is not in it + if err != nil { + t.Fatalf("New: %v", err) + } + if !p.knownSource("foreign") { + t.Error("transplanted archive should be addressable") + } + if got := p.ArchiveName("foreign"); got != "Le Journal" { + t.Errorf("ArchiveName = %q, want the transplanted self-describing label", got) + } + if _, ok := p.ArchiveIndex()["foreign"]; !ok { + t.Error("transplanted archive should appear in the archive index") + } +} + func TestServe_CacheInvalidatesOnArchiveOverwrite(t *testing.T) { - p, arch, date := newTestEngine(t, 64, "a") + p, arch, date := newTestEngine(t, "a") res, err := p.RenderFor(context.Background(), "a") if err != nil { @@ -185,7 +320,7 @@ func TestServe_CacheInvalidatesOnArchiveOverwrite(t *testing.T) { // A corrected edition is re-posted: same day, different pixels. Bump the // artifact's mtime well past the cached PNG's so freshness is unambiguous. if _, err := arch.Put("a", source.Edition{ - Date: date, Media: source.MediaImage, Data: uniformPNG(t, 32, 48, 255), // white + Date: date, Media: source.MediaImage, Data: uniformPNG(t, 255), // white }); err != nil { t.Fatalf("archive.Put overwrite: %v", err) } @@ -205,7 +340,7 @@ func TestServe_CacheInvalidatesOnArchiveOverwrite(t *testing.T) { } func TestServe_CacheKeyIncludesMasterWidth(t *testing.T) { - p, _, _ := newTestEngine(t, 64, "a") + p, _, _ := newTestEngine(t, "a") if _, err := p.RenderFor(context.Background(), "a"); err != nil { t.Fatalf("RenderFor: %v", err) } @@ -481,7 +616,7 @@ func TestServe_RasterizationsAreGloballyBounded(t *testing.T) { } func TestServe_VariantCacheSkipsMasterDecode(t *testing.T) { - p, arch, _ := newTestEngine(t, 64, "a") + p, arch, _ := newTestEngine(t, "a") first, err := p.RenderFor(context.Background(), "a", RenderOptions{OutputWidth: 48}) if err != nil { diff --git a/pkg/broadsheet/rotation_test.go b/pkg/broadsheet/rotation_test.go index 20a8d3a..f1e4997 100644 --- a/pkg/broadsheet/rotation_test.go +++ b/pkg/broadsheet/rotation_test.go @@ -29,7 +29,7 @@ func newRotationEngine(t *testing.T, withContent map[string]bool, ids ...string) srcs = append(srcs, Source{ID: id, DisplayName: id}) if withContent[id] { if _, err := arch.Put(id, source.Edition{ - Date: date, Media: source.MediaImage, Data: uniformPNG(t, 32, 48, 128), + Date: date, Media: source.MediaImage, Data: uniformPNG(t, 128), }); err != nil { t.Fatalf("archive.Put(%s): %v", id, err) }