diff --git a/.github/workflows/nextjs.yml b/.github/workflows/nextjs.yml index 57b3be2..b22ba00 100644 --- a/.github/workflows/nextjs.yml +++ b/.github/workflows/nextjs.yml @@ -4,7 +4,7 @@ on: push: branches: ["master", "main"] workflow_run: - workflows: ["Nightly ETL Scraper"] + workflows: ["Manual ETL Scraper"] types: - completed workflow_dispatch: diff --git a/.github/workflows/scraper.yml b/.github/workflows/scraper.yml index 6077da4..3d6ef71 100644 --- a/.github/workflows/scraper.yml +++ b/.github/workflows/scraper.yml @@ -1,9 +1,6 @@ -name: Nightly ETL Scraper +name: Manual ETL Scraper on: - schedule: - # Run every day at 1:00 AM UTC - - cron: '0 1 * * *' workflow_dispatch: inputs: state_unverified_enrichment_limit: @@ -51,6 +48,7 @@ permissions: contents: read concurrency: + # Retain the existing group so a dispatch cannot overlap an older in-flight run. group: nightly-etl cancel-in-progress: false @@ -94,21 +92,14 @@ jobs: # Bounded state LittleSis enrichment (unverified; optional) STATE_UNVERIFIED_ENRICHMENT_LIMIT: ${{ github.event.inputs.state_unverified_enrichment_limit || vars.STATE_UNVERIFIED_ENRICHMENT_LIMIT }} STATE_UNVERIFIED_ENRICHMENT_OFFSET: ${{ github.event.inputs.state_unverified_enrichment_offset || vars.STATE_UNVERIFIED_ENRICHMENT_OFFSET }} - # The reviewed nightly schedule writes the same bounded snapshot after the - # successful production canary. Manual dispatches keep an explicit choice; - # any other event fails closed. - HOUSE_ROLL_CALL_WRITE_MODE: ${{ github.event_name == 'schedule' && 'enabled' || github.event_name == 'workflow_dispatch' && inputs.house_roll_call_write_mode || 'disabled' }} - # The reviewed nightly schedule writes the same bounded Senate snapshot - # after its successful canary and audit. Manual dispatches keep an explicit - # choice; any other event fails closed. - SENATE_ROLL_CALL_WRITE_MODE: ${{ github.event_name == 'schedule' && 'enabled' || github.event_name == 'workflow_dispatch' && inputs.senate_roll_call_write_mode || 'disabled' }} - # The reviewed nightly schedule writes the same bounded private metadata - # batch after its successful canary and audit. Manual dispatches retain an - # explicit choice; any other event fails closed. - CONGRESS_GOV_METADATA_WRITE_MODE: ${{ github.event_name == 'schedule' && 'enabled' || github.event_name == 'workflow_dispatch' && inputs.congress_gov_metadata_write_mode || 'disabled' }} + # Every manual dispatch requires an explicit bounded-write choice. The + # checked-in default and any unexpected event both fail closed. + HOUSE_ROLL_CALL_WRITE_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.house_roll_call_write_mode || 'disabled' }} + SENATE_ROLL_CALL_WRITE_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.senate_roll_call_write_mode || 'disabled' }} + CONGRESS_GOV_METADATA_WRITE_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.congress_gov_metadata_write_mode || 'disabled' }} # Current federal profile votes come from the official House/Senate paths. - # The unreliable person-filtered GovTrack compatibility crawl never runs on - # schedules and requires an explicit manual diagnostic choice. + # The unreliable person-filtered GovTrack compatibility crawl remains + # disabled by default and requires an explicit manual diagnostic choice. GOVTRACK_PROFILE_ENRICHMENT_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.govtrack_profile_enrichment_mode || 'disabled' }} # News API keys — all optional; aggregator falls back gracefully if absent CURRENTS_API_KEY: ${{ secrets.CURRENTS_API_KEY }} diff --git a/AGENTS.md b/AGENTS.md index 07b3c80..a0ea26d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,7 +40,7 @@ frontend is rebuilt/redeployed, NOT when the database changes:** dynamic routes at runtime. Its server component now fetches only the minimal profile header needed for the static shell; the contact card and profile tabs fetch live in the browser. -**The consequence:** after the nightly scraper writes new data, search and directory can link to +**The consequence:** after a scraper run writes new data, search and directory can link to `/profile?id=` immediately, and that live route can show the row without a frontend rebuild. The legacy pretty `/[politician_id]` SEO route for a brand-new row still appears only after a deploy, but its data spokes are live once the page exists. @@ -122,7 +122,7 @@ When contributing to this project, you must adhere strictly to these rules: ## 🚀 Next Steps & Outstanding Work - The active remaining roadmap is `docs/canonical_data_and_analytics_plan.md`. Phases 1 and 2 are implemented and the scraper identity resolver in Phase 3 is complete. Applied migrations - `0022` through `0036` establish deterministic identity, atomic source-profile writes, + `0022` through `0037` establish deterministic identity, atomic source-profile writes, provenance, person office terms, and private normalized House and Senate roll-call facts. Both official vote paths remain bounded, identifier-only, and isolated from legacy `voting_records`; the versioned `0031` read RPC combines presentation-safe @@ -133,9 +133,9 @@ When contributing to this project, you must adhere strictly to these rules: `0035` approves only the observed bounded detail path and adds private source-record-backed measure facts, exact official-roll-call links, and one atomic service-role writer without raw JSON, legacy vote writes, or a public read path. `0036` records the successful manual canary - and private audit before the reviewed workflow enables that same bounded path for schedules. - The current `0037` slice adds a narrow measure-aware public voting-record RPC without opening - those private tables or changing the v2 vote contract. + and private audit before the reviewed workflow enabled that same bounded path for schedules. + `0037` adds a narrow measure-aware public voting-record RPC without opening those private + tables or changing the v2 vote contract; it is applied and live-validated. - Monitor and resolve quarantined identity candidates instead of weakening the pre-write boundary. A person can have federal, state, and local roles over time; those roles must not become separate canonical people or be flattened into one office field. @@ -158,9 +158,13 @@ When contributing to this project, you must adhere strictly to these rules: each completed the exact 18-detail / 18-measure / 43-link / 40-roll-call contract with healthy writes; the post-observation live audit found zero violations. The first post-PR-109 scheduled run confirmed that the legacy person-filtered GovTrack crawl had zero attempts or failures while - the official House, Senate, and Congress.gov paths stayed healthy. The active slice is migration - `0037_congress_gov_measure_read_surface.sql` plus the live Voting Record integration. After - merge, apply `0037` before the next schedule and validate v3/v2 base-row equivalence, exact - measure arrays, browser ACLs, pagination, and representative House/Senate profiles. It requires - no new key or scraper source request. Future historical GovTrack refresh should use a separately - bounded checkpointed backfill. + the official House, Senate, and Congress.gov paths stayed healthy. Migration `0037` is now + applied: v3/v2 base-row equivalence, all 43 exact measure links across 40 roll calls, browser + ACLs, pagination, and representative House/Senate profiles were validated live. The scraper + cron is intentionally paused during active development. Use only the manual workflow, keep the + state LittleSis fields blank and GovTrack profile mode disabled for routine runs, explicitly + enable the reviewed House/Senate/Congress.gov modes when refreshing those facts, and never + overlap runs. Allow at least one hour between full runs and ordinarily no more than three per + UTC day while OpenStates is active; do not restore the schedule without a reviewed change and + maintainer decision. Future historical GovTrack refresh should use a separately bounded, + checkpointed backfill. diff --git a/README.md b/README.md index 64d9210..9b3e727 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,8 @@ classifies, and displays publicly available information about U.S. politicians a Federal, State, and Local levels, presenting it as a clean, read-only encyclopedia. The project follows a **decoupled, zero-cost architecture**: a Python ETL pipeline pushes -data into Supabase on a schedule, and a statically-exported Next.js frontend reads from it. +data into Supabase through reviewed manual runs while the product is under active development, +and a statically-exported Next.js frontend reads from it. The render model is **static export with live client data**: home/search, `/directory`, `/profile?id=`, and the profile contact / financial / donor / voting / media / connections spokes query Supabase **live in the browser**. The legacy pretty @@ -23,7 +24,7 @@ profile spokes now hydrate from the browser once the page exists. Read ┌────────────────────┐ ┌──────────────────┐ ┌─────────────────────┐ │ Scraper (Python) │ ──▶ │ Supabase (Postgres)│ ◀── │ Frontend (Next.js) │ │ GitHub Actions │ │ REST API │ │ Static export → │ -│ (nightly sync) │ │ │ │ GitHub Pages │ +│ (manual sync) │ │ │ │ GitHub Pages │ └────────────────────┘ └──────────────────┘ └─────────────────────┘ ``` @@ -55,7 +56,7 @@ notes. │ └── extractors/ # Per-source extractors (fec, federal, govtrack, news, …) ├── migrations/ # SQL migrations ├── schema.sql # Database schema blueprint -└── .github/workflows/ # nextjs.yml (deploy) + scraper.yml (nightly ETL) +└── .github/workflows/ # nextjs.yml (deploy) + scraper.yml (manual ETL) ``` --- @@ -156,12 +157,12 @@ financial disclosures are not yet covered. | `FEC_API_KEY` | scraper | no | data.gov key for campaign-donor enrichment | | `OPENSTATES_API_KEY` | scraper | no | OpenStates key for state roll-call votes | | `CONGRESS_GOV_API_KEY` | scraper | no | Free data.gov key for bounded exact-detail Congress.gov metadata | -| `GOVTRACK_PROFILE_ENRICHMENT_MODE` | scraper | no | `disabled` by default and on schedules; explicit manual diagnostic opt-in for the legacy person-filtered crawl | +| `GOVTRACK_PROFILE_ENRICHMENT_MODE` | scraper | no | `disabled` by default; explicit manual diagnostic opt-in for the legacy person-filtered crawl | | `STATE_UNVERIFIED_ENRICHMENT_LIMIT` | scraper | no | Bounded count of state profiles to enrich via LittleSis | | `STATE_UNVERIFIED_ENRICHMENT_OFFSET` | scraper | no | Zero-based start offset for rotating state LittleSis batches | | `HOUSE_ROLL_CALL_WRITE_MODE` | scraper | no | `disabled` by default; `enabled` opts into the separately DB-gated House RPC | -| `SENATE_ROLL_CALL_WRITE_MODE` | scraper | no | `disabled` by default; reviewed nightly workflow schedules explicitly select `enabled` for the DB-gated Senate RPC | -| `CONGRESS_GOV_METADATA_WRITE_MODE` | scraper | no | `disabled` by default; reviewed nightly schedules explicitly select `enabled` after migration `0036` | +| `SENATE_ROLL_CALL_WRITE_MODE` | scraper | no | `disabled` by default; `enabled` opts into the separately DB-gated Senate RPC | +| `CONGRESS_GOV_METADATA_WRITE_MODE` | scraper | no | `disabled` by default; `enabled` opts into the DB-gated private metadata writer | | `CURRENTS_API_KEY` | scraper | no | News tier 1 | | `NEWSDATA_API_KEY` | scraper | no | News tier 2 (requires attribution) | | `THENEWSAPI_KEY` | scraper | no | News tier 3 credential | @@ -172,7 +173,7 @@ must explicitly set `ALLOW_MOCK_BUILD=true`. Production builds and runtime pages visibly instead of presenting fixtures as live data. The news aggregator works with no keys at all (it degrades to keyless GDELT URL discovery). -The legacy per-politician GovTrack REST crawl is no longer part of nightly or default runs. +The legacy per-politician GovTrack REST crawl is no longer part of default runs. `GOVTRACK_PROFILE_ENRICHMENT_MODE=enabled` is available only as an explicit manual diagnostic choice. This switch controls the expensive person-filtered compatibility endpoint; it does not disable the bounded vote-specific GovTrack comparisons that protect the official House Clerk and @@ -192,8 +193,8 @@ only through the trusted `congress-legislators` LIS-to-Bioguide crosswalk, and r normalized snapshot in memory. `SENATE_ROLL_CALL_WRITE_MODE=enabled` permits a run to call the atomic Senate RPC only when the complete bounded listing, every official XML document, every exact identity, every GovTrack comparison snapshot, and source health are complete. Code, -example-environment, and manual-input defaults remain `disabled`. After the reviewed canary and -audit, the nightly `schedule` explicitly selects `enabled`; unknown events still fail closed. +example-environment, and manual-input defaults remain `disabled`. The current manual-only +workflow requires an explicit choice on every dispatch; unexpected events still fail closed. Congress.gov metadata uses exact bill and amendment identifiers already present in those two official snapshots. It never calls collection endpoints and caps each run at 100 distinct detail @@ -203,9 +204,9 @@ measure facts, and exact roll-call links only when both upstream official snapsh detail-fetch counters reconcile exactly. The successful manual canary wrote 18 measures and 43 exact links with healthy fetch and write trackers; its post-canary audit found zero provenance, fact, link, ACL, or legacy-isolation violations, and an exact replay changed no row image or -transaction ID. Migration `0036` records that review and the nightly `schedule` explicitly -selects `enabled`; code, example-environment, and manual-input defaults remain `disabled`, and -unknown events fail closed. The path retains no raw JSON and writes no legacy votes. Migration +transaction ID. Migration `0036` records the historical schedule-enablement review; the current +manual-only workflow still defaults this mode to `disabled`, requires an explicit choice, and +fails closed for unexpected events. The path retains no raw JSON and writes no legacy votes. Migration `0037` adds the first narrow public presentation path: a versioned voting-record RPC delegates all person resolution, filtering, deduplication, ordering, and pagination to the proven v2 RPC, then decorates only official rows with at most 100 exact linked measures. It exposes identifiers, @@ -264,12 +265,12 @@ table/column access to read-only, and preserves only controlled security-definer A null-safe House event-prefix namespace constraint also prevents the unrelated generic profile and retirement RPCs from colliding with House provenance. The migration then enables the reviewed source-catalog database gates atomically. -Runtime writes still default to `disabled` in code and the example environment. After the -successful bounded production canary and post-canary database audit, the GitHub Actions nightly -schedule explicitly passes `enabled` for the same bounded write path. Manual workflow runs keep -their required `disabled`/`enabled` choice with `disabled` as the default, and any unrecognized -event fails closed. The database gates remain independently disableable. This path never writes -legacy `voting_records`. +Runtime writes still default to `disabled` in code and the example environment. The successful +bounded production canary and post-canary audit previously permitted scheduled writes; during +active development, the cron trigger is paused and manual workflow runs keep their required +`disabled`/`enabled` choice with `disabled` as the default. Any unrecognized event fails closed. +The database gates remain independently disableable. This path never writes legacy +`voting_records`. Apply migration `0027` with `ON_ERROR_STOP=1` and without psql's external `--single-transaction` option. Its two checked-in transactions are the database-enforced cutover: @@ -300,11 +301,28 @@ Both pipelines run from GitHub Actions: connections. The legacy pretty `/[politician_id]` route availability is still tied to static generation, so brand-new rows should be linked through `/profile?id=` until a deploy creates the SEO route. - The deploy is wired to re-run automatically after a *successful* nightly ETL via a + The deploy is wired to re-run automatically after a *successful* manual ETL via a `workflow_run` trigger. (A failed scraper run does **not** trigger the deploy, so the live site keeps the last good build rather than shipping nothing.) -- **`.github/workflows/scraper.yml`** — runs the Python ETL on a nightly schedule, writing - fresh data into Supabase. +- **`.github/workflows/scraper.yml`** — runs the Python ETL only when explicitly dispatched, + writing fresh data into Supabase. It has no cron trigger while active development is underway. + +### Manual scraper operations + +The **Manual ETL Scraper** workflow serializes all dispatches and allows a maximum of four hours +per run. For the reviewed full federal refresh, leave the two state LittleSis fields blank, set +the House, Senate, and Congress.gov write modes to `enabled`, and leave the legacy GovTrack +profile mode `disabled`. Do not queue a second run while one is active. + +Repeated runs share upstream account quotas even though each process resets its local counters. +Keep at least one hour between full runs and, while the OpenStates key is active, use no more than +three full runs in a UTC day unless its provider dashboard shows enough remaining quota. The main +limits to watch in `ETL_SUMMARY_JSON` are OpenFEC's 900-request local cap (against an approximately +1,000-request/hour free allowance), OpenStates' 450-request local cap with 10/minute pacing +(against an approximately 500-request/day free allowance), and the account-specific news quotas. +The official House and Senate windows are each capped at 25 roll calls, Congress.gov at 100 exact +detail references (18 in recent successful runs), and the legacy person-filtered GovTrack crawl +should remain disabled. Local caps are safety ceilings, not evidence of remaining upstream quota. ### Interpreting ETL identity-health output @@ -390,8 +408,9 @@ only runs the ETL and `nextjs.yml` only builds. official-roll-call links, installs one bounded atomic service-role writer, and advances scraper preflight to `0035`. It retains no raw API JSON and creates no browser read path. `0036` validates and records the successful manual canary and private database audit, advances - scraper preflight, and permits the nightly workflow to select the same bounded writer. Runtime - and manual-input defaults remain disabled, and unknown events fail closed. `0037` creates the + scraper preflight, and historically permitted the scheduled workflow to select the same + bounded writer. The current cron pause leaves runtime and manual-input defaults disabled, and + unknown events fail closed. `0037` creates the measure-aware `get_canonical_voting_records_v3` read RPC without changing v2, legacy votes, or scraper writers; it advances preflight so the new read contract cannot silently drift. @@ -408,7 +427,7 @@ maintainer review state. Use a new forward repair migration instead of editing l > 1. Apply the pending migrations (e.g. `0002`–`0007`) against the live database. If a > freshly-added column still isn't found right after, reload PostgREST's schema cache: > `NOTIFY pgrst, 'reload schema';` -> 2. **Run the Nightly ETL Scraper and confirm it succeeds** — look for `[+] Updated/Inserted +> 2. **Run the Manual ETL Scraper and confirm it succeeds** — look for `[+] Updated/Inserted > Hub` lines and **no** PGRST204 errors. This is the step that actually writes data; a > drifted run writes *nothing*. > 3. Re-run **Deploy Next.js to GitHub Pages** so the profile pages re-bake. A successful diff --git a/docs/canonical_data_and_analytics_plan.md b/docs/canonical_data_and_analytics_plan.md index 4a90959..8c8aef7 100644 --- a/docs/canonical_data_and_analytics_plan.md +++ b/docs/canonical_data_and_analytics_plan.md @@ -703,15 +703,21 @@ requests and 25 writes, the Senate shadow completed 77 of 77 requests and 25 wri Congress.gov path completed 18 of 18 details plus one healthy atomic write. The run reported no errors, so the profile-crawl retirement observation gate is complete. -The current presentation slice adds migration -`0037_congress_gov_measure_read_surface.sql` and client integration. The v3 RPC wraps the +The presentation slice added and applied migration +`0037_congress_gov_measure_read_surface.sql` plus its client integration. The v3 RPC wraps the unchanged v2 vote contract, aggregates exact measure links without duplicating vote rows, and -returns only bounded presentation-safe facts. The live client falls back to v2 only while 0037 -is not yet installed, while scraper preflight advances to the 0037 marker so deployment drift -cannot remain silent. After merge, apply 0037 before the next scheduled scraper and validate the -v3/v2 base-row equivalence, exact measure arrays, browser ACLs, pagination, and representative -House and Senate profiles. This slice adds no source request, credential, writer, or legacy vote -mutation. +returns only bounded presentation-safe facts. Live validation confirmed v3/v2 base-row +equivalence, all 43 exact measure links across 40 roll calls, closed private-table ACLs, browser +access, pagination, and representative House and Senate profiles. Scraper preflight now requires +the 0037 marker, so deployment drift cannot remain silent. This slice added no source request, +credential, writer, or legacy vote mutation. + +During active site development the cron trigger is intentionally paused. The ETL is dispatched +manually, remains serialized, and keeps every write-mode input disabled by default. Routine full +refreshes explicitly enable only the reviewed House, Senate, and Congress.gov paths, keep the +legacy person-filtered GovTrack crawl disabled, and leave state LittleSis enrichment blank. Allow +at least one hour between full runs and ordinarily no more than three per UTC day while the +OpenStates key is active; restore a schedule only through a separately reviewed change. Broader candidate triage, including the FCC/GSA context pair seeded by `0024`, stays separate from this vote slice; historical identity-review queue cleanup is deferred to Phase 6. Do not diff --git a/docs/source_usage_policy.md b/docs/source_usage_policy.md index 6986842..b26af0b 100644 --- a/docs/source_usage_policy.md +++ b/docs/source_usage_policy.md @@ -101,7 +101,7 @@ SHA-256 for provenance but never retains raw JSON. The nonblocking fetch tracker attempts, successes, failures, skips, coverage, and breaker state; it retries a server failure once and stops on authentication, quota, or identity conflicts. Removing `CONGRESS_GOV_API_KEY` disables the path without affecting the official vote writers or their -retained facts. Shared `DEMO_KEY` use is refused by the scheduled pipeline. +retained facts. Shared `DEMO_KEY` use is refused by the ETL pipeline. Migration `0034_congress_gov_metadata_shadow_contract.sql` corrects the seeded API base to `https://api.congress.gov/v3/`, reserves the official `congress-gov` source namespace, and @@ -129,9 +129,10 @@ returned 18 measures and 43 links while changing neither stored row images nor t Migration `0036_congress_gov_scheduled_enablement.sql` validates and records that evidence before advancing scraper preflight. `CONGRESS_GOV_METADATA_WRITE_MODE` remains `disabled` by default in -code, the example environment, and manual workflow input. The reviewed nightly `schedule` -explicitly selects `enabled` for the same bounded path; unknown events fail closed, and any -failed enabled write remains run-blocking. +code, the example environment, and manual workflow input. The reviewed schedule previously +selected `enabled` for the same bounded path; the cron trigger is currently paused and every +manual dispatch requires an explicit choice. Unknown events fail closed, and any failed enabled +write remains run-blocking. Migration `0037_congress_gov_measure_read_surface.sql` is the separately reviewed presentation gate. Its versioned `get_canonical_voting_records_v3` RPC delegates canonical person resolution, @@ -146,7 +147,7 @@ The client independently validates the exact measure key and safe HTTP URL befor raw JSON, payload hash, private metadata, writer capability, or new source request crosses this read boundary. -### GovTrack legacy profile enrichment (retained history; scheduled fetch disabled) +### GovTrack legacy profile enrichment (retained history; disabled by default) The legacy profile spoke calls GovTrack's person-filtered `vote_voter` endpoint by the stable GovTrack ID supplied by `congress-legislators`. It never joins by name. Its retained rows still @@ -162,14 +163,14 @@ recorded nine failures and 498 breaker skips after 30 successes. In both runs, t bounded vote-specific GovTrack comparisons used by the official House and Senate writers were healthy, and those authoritative writes completed successfully. -`GOVTRACK_PROFILE_ENRICHMENT_MODE` therefore defaults to `disabled`, scheduled events always -select `disabled`, and unknown events fail closed. A manual workflow dispatch may explicitly -select `enabled` for diagnostics. Disabling the profile crawl does not delete legacy +`GOVTRACK_PROFILE_ENRICHMENT_MODE` therefore defaults to `disabled`, and unknown events fail +closed. A manual workflow dispatch may explicitly select `enabled` for diagnostics. Disabling +the profile crawl does not delete legacy `voting_records`, change the official-vote read RPC, or affect the vote-specific reconciliation requests. Any future historical refresh should use a separately reviewed bounded bulk/backfill path with its own recovery checkpoint rather than restoring the 537-request nightly crawl. -### Senate roll-call XML (approved; database-gated, bounded scheduled writes) +### Senate roll-call XML (approved; database-gated, bounded manual writes) The U.S. Senate publishes an [XML record for each roll call](https://www.senate.gov/legislative/LIS/roll_call_votes/) through the Senate Legislative Information System. The integration fetches at most the 25 most @@ -261,10 +262,10 @@ errors. Its service-role exact replay returned the complete member count and cha full row images nor transaction IDs. `SENATE_ROLL_CALL_WRITE_MODE` remains `disabled` by default in code, the example environment, -and the manual workflow input. The reviewed nightly `schedule` explicitly selects `enabled` for -the same bounded path; manual dispatches retain their explicit choice and unknown events fail -closed. Disabling the runtime control or either database gate preserves shadow-only behavior -and the last valid rows. +and the manual workflow input. The reviewed workflow previously selected `enabled` on nightly +schedules; that cron trigger is currently paused during active development. Manual dispatches +retain their explicit choice and unknown events fail closed. Disabling the runtime control or +either database gate preserves shadow-only behavior and the last valid rows. ### House Clerk roll-call XML (approved; database-gated, runtime opt-in) @@ -345,7 +346,8 @@ without reserving unrelated Clerk record families. The same transaction then ena reviewed database gate rows. `HOUSE_ROLL_CALL_WRITE_MODE` nevertheless defaults to `disabled` in code and the example environment. After the successful bounded production canary and post-canary database audit, the -GitHub Actions nightly schedule explicitly selects `enabled` for the same bounded path. Manual -runs retain a required `disabled`/`enabled` choice whose default is `disabled`, and unrecognized -events fail closed. Disabling either the runtime control or either database gate preserves -shadow-only behavior and the last valid rows. +GitHub Actions schedule previously selected `enabled` for the same bounded path; that cron +trigger is currently paused during active development. Manual runs retain a required +`disabled`/`enabled` choice whose default is `disabled`, and unrecognized events fail closed. +Disabling either the runtime control or either database gate preserves shadow-only behavior and +the last valid rows. diff --git a/scraper/extractors/govtrack.py b/scraper/extractors/govtrack.py index ac97d3f..656825e 100644 --- a/scraper/extractors/govtrack.py +++ b/scraper/extractors/govtrack.py @@ -3,10 +3,10 @@ Legacy voting-record enrichment from the free GovTrack API (no key required), joined by the GovTrack person ID carried in politicians.external_ids["govtrack"] (from the -congress-legislators crosswalk) — no fuzzy name matching. The default and scheduled -pipeline leave this expensive person-filtered compatibility path disabled now that -bounded official House Clerk and Senate LIS facts serve current federal votes. An -explicit manual diagnostic opt-in can still call it without deleting retained history. +congress-legislators crosswalk) — no fuzzy name matching. The pipeline leaves this +expensive person-filtered compatibility path disabled by default now that bounded +official House Clerk and Senate LIS facts serve current federal votes. An explicit +manual diagnostic opt-in can still call it without deleting retained history. This legacy VERIFIED spoke lands in voting_records when explicitly enabled. diff --git a/scraper/source_health_config.py b/scraper/source_health_config.py index 02523fa..1a3b5a5 100644 --- a/scraper/source_health_config.py +++ b/scraper/source_health_config.py @@ -48,8 +48,8 @@ def build_source_health_trackers(summary: ETLRunSummary) -> dict: max_failure_seconds=OPENFEC_MAX_FAILURE_SECONDS, affects_run=False, ), - # The legacy person-filtered profile crawl is disabled for defaults and - # schedules. When a manual diagnostic explicitly enables it, failures remain + # The legacy person-filtered profile crawl is disabled by default. When a + # manual diagnostic explicitly enables it, failures remain # observable but cannot invalidate healthy official House/Senate writes. "govtrack": summary.source_tracker( "govtrack", min_attempts_for_rate=10, affects_run=False diff --git a/scraper/tests/test_congress_gov_metadata_runtime.py b/scraper/tests/test_congress_gov_metadata_runtime.py index c23aa3b..86535f1 100644 --- a/scraper/tests/test_congress_gov_metadata_runtime.py +++ b/scraper/tests/test_congress_gov_metadata_runtime.py @@ -87,7 +87,7 @@ def test_write_mode_defaults_to_disabled_and_requires_explicit_enabled(self): {"CONGRESS_GOV_METADATA_WRITE_MODE": invalid} ) - def test_checked_in_configuration_enables_only_reviewed_schedule(self): + def test_checked_in_configuration_requires_manual_opt_in(self): workflow = (_REPO_ROOT / ".github" / "workflows" / "scraper.yml").read_text( encoding="utf-8" ) @@ -103,7 +103,6 @@ def test_checked_in_configuration_enables_only_reviewed_schedule(self): ) self.assertIsNotNone(expression) self.assertEqual( - "github.event_name == 'schedule' && 'enabled' || " "github.event_name == 'workflow_dispatch' " "&& inputs.congress_gov_metadata_write_mode || 'disabled'", expression.group(1), @@ -112,8 +111,6 @@ def test_checked_in_configuration_enables_only_reviewed_schedule(self): self.assertIn("CONGRESS_GOV_METADATA_WRITE_MODE=disabled", example_env) def resolve(event_name, manual_input, _repository_variable): - if event_name == "schedule": - return "enabled" if event_name == "workflow_dispatch": return manual_input or "disabled" return "disabled" @@ -126,8 +123,8 @@ def resolve(event_name, manual_input, _repository_variable): resolve("workflow_dispatch", "enabled", "disabled"), "enabled", ) - self.assertEqual(resolve("schedule", None, "disabled"), "enabled") - self.assertEqual(resolve("schedule", None, "enabled"), "enabled") + self.assertEqual(resolve("schedule", None, "disabled"), "disabled") + self.assertEqual(resolve("schedule", None, "enabled"), "disabled") self.assertEqual(resolve("push", None, "enabled"), "disabled") def test_disabled_mode_never_calls_loader(self): diff --git a/scraper/tests/test_house_roll_call_runtime.py b/scraper/tests/test_house_roll_call_runtime.py index 51c7ee0..6776af5 100644 --- a/scraper/tests/test_house_roll_call_runtime.py +++ b/scraper/tests/test_house_roll_call_runtime.py @@ -90,7 +90,7 @@ def test_write_mode_accepts_only_explicit_enabled_value(self): with self.subTest(invalid=invalid), self.assertRaises(ValueError): house_roll_call_write_mode({"HOUSE_ROLL_CALL_WRITE_MODE": invalid}) - def test_checked_in_runtime_configuration_enables_reviewed_schedule(self): + def test_checked_in_runtime_configuration_requires_manual_opt_in(self): workflow = (_REPO_ROOT / ".github" / "workflows" / "scraper.yml").read_text( encoding="utf-8" ) @@ -110,7 +110,6 @@ def test_checked_in_runtime_configuration_enables_reviewed_schedule(self): self.assertIsNotNone(expression) self.assertEqual( expression.group(1), - "github.event_name == 'schedule' && 'enabled' || " "github.event_name == 'workflow_dispatch' " "&& inputs.house_roll_call_write_mode || 'disabled'", ) @@ -118,16 +117,14 @@ def test_checked_in_runtime_configuration_enables_reviewed_schedule(self): self.assertIn("HOUSE_ROLL_CALL_WRITE_MODE=disabled", example_env) def resolve(event_name, manual_input, _repository_variable): - if event_name == "schedule": - return "enabled" if event_name == "workflow_dispatch": return manual_input or "disabled" return "disabled" self.assertEqual(resolve("workflow_dispatch", "disabled", "enabled"), "disabled") self.assertEqual(resolve("workflow_dispatch", "enabled", "disabled"), "enabled") - self.assertEqual(resolve("schedule", None, "disabled"), "enabled") - self.assertEqual(resolve("schedule", None, "enabled"), "enabled") + self.assertEqual(resolve("schedule", None, "disabled"), "disabled") + self.assertEqual(resolve("schedule", None, "enabled"), "disabled") self.assertEqual(resolve("push", None, "enabled"), "disabled") def test_disabled_mode_never_calls_the_loader(self): diff --git a/scraper/tests/test_manual_scraper_workflow.py b/scraper/tests/test_manual_scraper_workflow.py new file mode 100644 index 0000000..f275438 --- /dev/null +++ b/scraper/tests/test_manual_scraper_workflow.py @@ -0,0 +1,37 @@ +import unittest +from pathlib import Path + + +_REPO_ROOT = Path(__file__).resolve().parents[2] + + +class ManualScraperWorkflowTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.scraper_workflow = ( + _REPO_ROOT / ".github" / "workflows" / "scraper.yml" + ).read_text(encoding="utf-8") + cls.deploy_workflow = ( + _REPO_ROOT / ".github" / "workflows" / "nextjs.yml" + ).read_text(encoding="utf-8") + + def test_scraper_has_only_a_manual_trigger(self): + self.assertIn("name: Manual ETL Scraper", self.scraper_workflow) + self.assertIn("\non:\n workflow_dispatch:\n", self.scraper_workflow) + self.assertNotIn("\n schedule:", self.scraper_workflow) + self.assertNotIn("cron:", self.scraper_workflow) + + def test_manual_runs_remain_serial_and_bounded(self): + self.assertIn("group: nightly-etl", self.scraper_workflow) + self.assertIn("cancel-in-progress: false", self.scraper_workflow) + self.assertIn("timeout-minutes: 240", self.scraper_workflow) + + def test_successful_manual_etl_still_triggers_a_frontend_deploy(self): + self.assertIn( + 'workflows: ["Manual ETL Scraper"]', + self.deploy_workflow, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scraper/tests/test_senate_roll_call_runtime.py b/scraper/tests/test_senate_roll_call_runtime.py index 7d6eb05..4aeed32 100644 --- a/scraper/tests/test_senate_roll_call_runtime.py +++ b/scraper/tests/test_senate_roll_call_runtime.py @@ -98,7 +98,7 @@ def test_write_mode_accepts_only_explicit_enabled_value(self): {"SENATE_ROLL_CALL_WRITE_MODE": invalid} ) - def test_checked_in_runtime_configuration_enables_only_reviewed_schedule(self): + def test_checked_in_runtime_configuration_requires_manual_opt_in(self): workflow = (_REPO_ROOT / ".github" / "workflows" / "scraper.yml").read_text( encoding="utf-8" ) @@ -116,7 +116,6 @@ def test_checked_in_runtime_configuration_enables_only_reviewed_schedule(self): self.assertIsNotNone(expression) self.assertEqual( expression.group(1), - "github.event_name == 'schedule' && 'enabled' || " "github.event_name == 'workflow_dispatch' " "&& inputs.senate_roll_call_write_mode || 'disabled'", ) @@ -124,16 +123,14 @@ def test_checked_in_runtime_configuration_enables_only_reviewed_schedule(self): self.assertIn("SENATE_ROLL_CALL_WRITE_MODE=disabled", example_env) def resolve(event_name, manual_input, _repository_variable): - if event_name == "schedule": - return "enabled" if event_name == "workflow_dispatch": return manual_input or "disabled" return "disabled" self.assertEqual(resolve("workflow_dispatch", "disabled", "enabled"), "disabled") self.assertEqual(resolve("workflow_dispatch", "enabled", "disabled"), "enabled") - self.assertEqual(resolve("schedule", None, "disabled"), "enabled") - self.assertEqual(resolve("schedule", None, "enabled"), "enabled") + self.assertEqual(resolve("schedule", None, "disabled"), "disabled") + self.assertEqual(resolve("schedule", None, "enabled"), "disabled") self.assertEqual(resolve("push", None, "enabled"), "disabled") def test_disabled_mode_never_calls_the_loader(self): diff --git a/scraper/tests/test_senate_roll_call_source_review.py b/scraper/tests/test_senate_roll_call_source_review.py index 0ccc9ec..12666a2 100644 --- a/scraper/tests/test_senate_roll_call_source_review.py +++ b/scraper/tests/test_senate_roll_call_source_review.py @@ -172,7 +172,7 @@ def test_requires_prior_history_and_records_a_forward_only_marker(self): def test_public_docs_record_the_bounded_approval_and_next_gate(self): self.assertIn( - "Senate roll-call XML (approved; database-gated, bounded scheduled writes)", + "Senate roll-call XML (approved; database-gated, bounded manual writes)", self.policy, ) self.assertIn("4,996 exact LIS matches", self.policy)