diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index bfcfd04f..83953f08 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -5,6 +5,7 @@ import { congressConfig } from "./scrapers/congress.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; import { scotusConfig } from "./scrapers/scotus.config.js"; +import { texasCurrentElectionConfig } from "./scrapers/texas-current-election.config.js"; export const scraperContracts: readonly ScraperEnvContract[] = [ federalregisterConfig, @@ -12,4 +13,5 @@ export const scraperContracts: readonly ScraperEnvContract[] = [ scotusConfig, sccCvigConfig, caSosStatementsConfig, + texasCurrentElectionConfig, ]; diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index d4ab8d4d..47743ba5 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -4,6 +4,7 @@ import { congress } from "./scrapers/congress.js"; import { federalregister } from "./scrapers/federalregister.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; import { scotus } from "./scrapers/scotus.js"; +import { texasCurrentElection } from "./scrapers/texas-current-election.js"; export const scrapers: readonly Scraper[] = [ federalregister, @@ -11,4 +12,5 @@ export const scrapers: readonly Scraper[] = [ scotus, sccCvig, caSosStatements, + texasCurrentElection, ]; diff --git a/apps/scraper/src/scrapers/texas-current-election.config.ts b/apps/scraper/src/scrapers/texas-current-election.config.ts new file mode 100644 index 00000000..4366309a --- /dev/null +++ b/apps/scraper/src/scrapers/texas-current-election.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const texasCurrentElectionConfig = { + id: "texas-current-election", + name: "Texas SOS/TLC current election cycle", + source: "Texas Secretary of State and Texas Legislative Council", + environment: { + required: ["POSTGRES_URL"], + optional: ["TX_SOS_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/texas-current-election.test.ts b/apps/scraper/src/scrapers/texas-current-election.test.ts new file mode 100644 index 00000000..0c546f17 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-current-election.test.ts @@ -0,0 +1,17 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { discoverLatestTlcAnalysis } from "./texas-current-election.js"; + +void test("TLC discovery chooses the newest full report and skips condensed PDFs", () => { + const html = ` + November 7, 2023 + Condensed + 2025 + `; + assert.deepEqual(discoverLatestTlcAnalysis(html), { + year: 2025, + title: "2025", + url: "https://tlc.texas.gov/docs/amendments/analyses25.pdf", + }); +}); diff --git a/apps/scraper/src/scrapers/texas-current-election.ts b/apps/scraper/src/scrapers/texas-current-election.ts new file mode 100644 index 00000000..6a2dc07b --- /dev/null +++ b/apps/scraper/src/scrapers/texas-current-election.ts @@ -0,0 +1,285 @@ +/** + * Current-cycle Texas statewide election ingestion. + * + * The SOS Civix application publishes base64-wrapped JSON for election + * discovery, contests/results, reporting status, and county totals. TLC's + * publications page links the latest cycle-specific constitutional-amendment + * analysis PDF. This scraper discovers both rather than constructing year URLs, + * parses them deterministically, and stores separate provider snapshots. + */ + +import { createHash } from "node:crypto"; +import * as cheerio from "cheerio"; +import { getDocumentProxy } from "unpdf"; + +import type { + TexasElectionDefinition, + TexasSosElection, + TexasSosSnapshotData, + TexasTlcSnapshotData, + TlcTextPage, +} from "@acme/api/lib/texas-election-data"; +import { + parseTexasSosDiscovery, + parseTexasSosElection, + parseTexasTlcAnalysis, + TEXAS_CURRENT_SCOPE, + TEXAS_RESULTS_URL, + TEXAS_SOS_PROVIDER, + TEXAS_TLC_PROVIDER, + TEXAS_TLC_PUBLICATIONS_URL, +} from "@acme/api/lib/texas-election-data"; +import { db } from "@acme/db/client"; +import { ElectionSourceSnapshot } from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { texasCurrentElectionConfig } from "./texas-current-election.config.js"; + +const logger = createLogger("texas-current-election"); +const API_BASE = + "https://goelect.txelections.civixapps.com/api-ivis-system/api/s3/enr"; +const USER_AGENT = + "Mozilla/5.0 (compatible; BillionCivicBot/1.0; +https://billion.app)"; + +interface TlcPublication { + year: number; + title: string; + url: string; +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +async function fetchJson(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "application/json", "User-Agent": USER_AGENT }, + }); + return response.json() as Promise; +} + +async function fetchText(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "text/html", "User-Agent": USER_AGENT }, + }); + return response.text(); +} + +async function fetchBytes(url: string): Promise { + const response = await fetchWithRetry(url, { + headers: { Accept: "application/pdf", "User-Agent": USER_AGENT }, + timeoutMs: 60_000, + }); + return new Uint8Array(await response.arrayBuffer()); +} + +/** Discover the newest full TLC analysis PDF from the publications index. */ +export function discoverLatestTlcAnalysis( + html: string, + baseUrl = TEXAS_TLC_PUBLICATIONS_URL, +): TlcPublication | null { + const $ = cheerio.load(html); + const candidates: TlcPublication[] = []; + $("a[href]").each((_index, element) => { + const href = $(element).attr("href"); + if (!href || /condensed/i.test(href)) return; + const match = /\/analyses(\d{2}|\d{4})\.pdf(?:$|[?#])/i.exec(href); + if (!match?.[1]) return; + const short = Number.parseInt(match[1], 10); + const year = short < 100 ? 2000 + short : short; + if (!Number.isInteger(year) || year > new Date().getFullYear()) return; + candidates.push({ + year, + title: + $(element).text().replace(/\s+/g, " ").trim() || + `Analyses of Proposed Constitutional Amendments (${year})`, + url: new URL(href, baseUrl).toString(), + }); + }); + return candidates.sort((a, b) => b.year - a.year)[0] ?? null; +} + +/** Extract each PDF page independently so citations retain real page numbers. */ +export async function extractTlcPages( + bytes: Uint8Array, +): Promise { + const pdf = await getDocumentProxy(bytes); + const pages: TlcTextPage[] = []; + for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber++) { + const page = await pdf.getPage(pageNumber); + const content = await page.getTextContent(); + const text = (content.items as Array<{ str?: string }>) + .map((item) => item.str?.trim() ?? "") + .filter(Boolean) + .join(" "); + pages.push({ page: pageNumber, text }); + } + return pages; +} + +async function loadSosElection( + definition: TexasElectionDefinition, +): Promise { + try { + const [election, counties] = await Promise.all([ + fetchJson(`${API_BASE}/election/${definition.id}`), + fetchJson(`${API_BASE}/election/countyInfo/${definition.id}`).catch( + () => undefined, + ), + ]); + return parseTexasSosElection(election, definition, counties); + } catch (error) { + logger.warn(`SOS election ${definition.id} could not be parsed:`, error); + return null; + } +} + +async function upsertSnapshot(input: { + cycleYear: number; + provider: string; + sourceVersion: string; + data: Record; + diagnostics: string[]; + sourceUrls: string[]; +}): Promise { + const contentHash = sha256(JSON.stringify(input.data)); + const now = new Date(); + await db + .insert(ElectionSourceSnapshot) + .values({ + jurisdiction: "TX", + cycleYear: input.cycleYear, + provider: input.provider, + scope: TEXAS_CURRENT_SCOPE, + sourceVersion: input.sourceVersion, + contentHash, + data: input.data, + diagnostics: input.diagnostics, + sourceUrls: input.sourceUrls, + fetchedAt: now, + }) + .onConflictDoUpdate({ + target: [ + ElectionSourceSnapshot.jurisdiction, + ElectionSourceSnapshot.cycleYear, + ElectionSourceSnapshot.provider, + ElectionSourceSnapshot.scope, + ], + set: { + sourceVersion: input.sourceVersion, + contentHash, + data: input.data, + diagnostics: input.diagnostics, + sourceUrls: input.sourceUrls, + fetchedAt: now, + }, + }); +} + +function selectedDefinitions( + definitions: TexasElectionDefinition[], + amendmentOnly: boolean, + remaining: number, +): TexasElectionDefinition[] { + const filtered = amendmentOnly + ? definitions.filter((definition) => + /constitutional amendment/i.test(definition.name), + ) + : definitions; + return filtered.slice(0, Math.max(0, remaining)); +} + +async function scrape(maxItems = 12): Promise { + logger.info("Discovering current Texas SOS elections and TLC analysis…"); + const [constantsRaw, publicationsHtml] = await Promise.all([ + fetchJson(`${API_BASE}/electionConstants`), + fetchText(TEXAS_TLC_PUBLICATIONS_URL), + ]); + const discovery = parseTexasSosDiscovery(constantsRaw); + const tlcPublication = discoverLatestTlcAnalysis(publicationsHtml); + const diagnostics: string[] = []; + if (!tlcPublication) + diagnostics.push("No current TLC amendment analysis PDF discovered"); + + let processed = 0; + const years = new Set([discovery.cycleYear]); + if (tlcPublication) years.add(tlcPublication.year); + for (const year of [...years].sort((a, b) => b - a)) { + const definitions = selectedDefinitions( + discovery.electionsByYear[year] ?? [], + year !== discovery.cycleYear, + maxItems - processed, + ); + const elections: TexasSosElection[] = []; + for (const definition of definitions) { + const election = await loadSosElection(definition); + processed++; + if (election) elections.push(election); + } + if (!elections.length) { + diagnostics.push(`No SOS election payloads parsed for ${year}`); + continue; + } + const data: TexasSosSnapshotData = { cycleYear: year, elections }; + await upsertSnapshot({ + cycleYear: year, + provider: TEXAS_SOS_PROVIDER, + sourceVersion: elections + .map((election) => election.sourceVersion) + .join(","), + data: data as unknown as Record, + diagnostics: diagnostics.filter((item) => item.includes(String(year))), + sourceUrls: [TEXAS_RESULTS_URL], + }); + logger.success( + `Texas SOS ${year}: persisted ${elections.length} election(s).`, + ); + } + + if (tlcPublication && processed < maxItems) { + try { + const bytes = await fetchBytes(tlcPublication.url); + const pages = await extractTlcPages(bytes); + const parsed: TexasTlcSnapshotData = parseTexasTlcAnalysis( + pages, + tlcPublication.url, + ); + parsed.publicationTitle = tlcPublication.title; + const tlcDiagnostics = [ + ...diagnostics.filter((item) => item.includes("TLC")), + ...parsed.measures.flatMap((measure) => + measure.diagnostics.map( + (item) => `Proposition ${measure.propositionNumber}: ${item}`, + ), + ), + ]; + if (!parsed.measures.length) { + tlcDiagnostics.push("TLC PDF contained no parseable propositions"); + } + await upsertSnapshot({ + cycleYear: parsed.cycleYear, + provider: TEXAS_TLC_PROVIDER, + sourceVersion: `tlc:${parsed.cycleYear}:${sha256(bytes).slice(0, 16)}`, + data: parsed as unknown as Record, + diagnostics: tlcDiagnostics, + sourceUrls: [TEXAS_TLC_PUBLICATIONS_URL, tlcPublication.url], + }); + logger.success( + `Texas TLC ${parsed.cycleYear}: persisted ${parsed.measures.length} proposition analyses.`, + ); + } catch (error) { + logger.warn( + "TLC analysis could not be parsed; SOS data remains available:", + error, + ); + } + } +} + +export const texasCurrentElection: Scraper = { + ...texasCurrentElectionConfig, + scrape: (options) => + scrape((options?.maxItems ?? Number(process.env.TX_SOS_MAX_ITEMS)) || 12), +}; diff --git a/docs/README.md b/docs/README.md index af74cdcc..e59da397 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,15 +4,16 @@ Start with [CONTRIBUTING.md](../CONTRIBUTING.md) for dev setup. These docs go de ## How the system works -| Doc | What it covers | -| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -| [Architecture overview](./architecture.md) | The big picture: system diagram, monorepo layout, build tooling, and why the key decisions were made | -| [Data layer](./data-layer.md) | Drizzle + Supabase Postgres, the schema (~20 tables), migrations | -| [API layer](./api.md) | The tRPC router, civic data integrations, request path, LLM provider | -| [Ballot-measure enrichment](./measure-enrichment.md) | How measure summaries are cross-validated across official sources, adapter by adapter | -| [Candidate enrichment](./candidate-enrichment.md) | The same cross-validation pattern applied to candidate bios/photos/contact info | -| [Scraper pipeline](./scraper.md) | The standalone content scraper: sources, change detection, AI generation | -| [Frontend apps](./frontend.md) | Expo mobile app, Next.js web, shared UI, cross-platform auth | +| Doc | What it covers | +| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| [Architecture overview](./architecture.md) | The big picture: system diagram, monorepo layout, build tooling, and why the key decisions were made | +| [Data layer](./data-layer.md) | Drizzle + Supabase Postgres, the schema (~20 tables), migrations | +| [API layer](./api.md) | The tRPC router, civic data integrations, request path, LLM provider | +| [Ballot-measure enrichment](./measure-enrichment.md) | How measure summaries are cross-validated across official sources, adapter by adapter | +| [Candidate enrichment](./candidate-enrichment.md) | The same cross-validation pattern applied to candidate bios/photos/contact info | +| [Texas current-election data](./texas-current-election.md) | Current-cycle SOS results/candidates and separately cited TLC amendment analyses | +| [Scraper pipeline](./scraper.md) | The standalone content scraper: sources, change detection, AI generation | +| [Frontend apps](./frontend.md) | Expo mobile app, Next.js web, shared UI, cross-platform auth | ## How to do things diff --git a/docs/api.md b/docs/api.md index 2f6381a2..31a36924 100644 --- a/docs/api.md +++ b/docs/api.md @@ -13,17 +13,17 @@ The API is a [tRPC v11](https://trpc.io/) router in `packages/api/`, served by N The root router (`packages/api/src/root.ts`) composes **nine** sub-routers: -| Router | Procedures (Q = query, M = mutation, πŸ”’ = protected) | -| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `auth` | `getSession` (Q), `getSecretMessage` (Q πŸ”’) | -| `civic` | `getElections`, `getVoterInfo`, `getRepresentatives`, `getRepresentativesEnriched` (all Q) β€” Google Civic + measure/candidate cross-validation | -| `places` | `autocomplete` (Q), `details` (M) β€” Google Places address autocomplete for the ballot lookup | -| `legistar` | `getLocalBills`, `getMeetings`, `getAgenda`, `getVotes`, `getBodies`, `getMeetingVotes` (all Q) β€” local councils | -| `openStates` | `searchBills`, `getBillDetails`, `getLegislators`, `getBillVotes` (all Q) β€” CA state legislature (Open States v3) | -| `content` | `getAll`, `getByType`, `getById` (all Q) β€” aggregates bill / government_content / court_case | -| `video` | `getInfinite` (Q) β€” cursor-paginated feed; converts `bytea` images to data URIs | -| `post` | `all`, `byId` (Q); `create`, `delete` (M πŸ”’) | -| `user` | preferences, blocked content, settings, profile, and saved-article CRUD (all πŸ”’) | +| Router | Procedures (Q = query, M = mutation, πŸ”’ = protected) | +| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `auth` | `getSession` (Q), `getSecretMessage` (Q πŸ”’) | +| `civic` | `getElections`, `getVoterInfo`, `getTexasCurrentElection`, `getRepresentatives`, `getRepresentativesEnriched` (all Q) β€” Google Civic + official current-cycle Texas data + enrichment | +| `places` | `autocomplete` (Q), `details` (M) β€” Google Places address autocomplete for the ballot lookup | +| `legistar` | `getLocalBills`, `getMeetings`, `getAgenda`, `getVotes`, `getBodies`, `getMeetingVotes` (all Q) β€” local councils | +| `openStates` | `searchBills`, `getBillDetails`, `getLegislators`, `getBillVotes` (all Q) β€” CA state legislature (Open States v3) | +| `content` | `getAll`, `getByType`, `getById` (all Q) β€” aggregates bill / government_content / court_case | +| `video` | `getInfinite` (Q) β€” cursor-paginated feed; converts `bytea` images to data URIs | +| `post` | `all`, `byId` (Q); `create`, `delete` (M πŸ”’) | +| `user` | preferences, blocked content, settings, profile, and saved-article CRUD (all πŸ”’) | ## Civic Data & External Sources @@ -37,6 +37,13 @@ Other live civic integrations: Ballot measures and candidates returned by `getVoterInfo` are run through cross-validation engines that merge multiple public-record sources by trust tier β€” see [Ballot-measure enrichment](./measure-enrichment.md) and [Candidate enrichment](./candidate-enrichment.md). Key/access setup for every source is in [Civic data source setup](./civic-data-sources.md). +`civic.getTexasCurrentElection` is an input-free reader over persisted current +Texas SOS/TLC snapshots. It returns current statewide/federal/district +candidates and results plus the latest constitutional-amendment analyses. SOS +facts and TLC explanation are separately cited. Historical election browsing is +intentionally not part of this procedure; see +[Texas current-election data](./texas-current-election.md). + ## LLM Provider `packages/api/src/lib/ai-provider.ts` exports a single swappable `llm` via the Vercel AI SDK: **Groq** when `GROQ_API_KEY` is set, then **OpenRouter** when `OPENROUTER_API_KEY` is present (using `OPENROUTER_MODEL`, default `deepseek/deepseek-v4-flash`), then **OpenAI `gpt-4o-mini`**, then deprecated direct **DeepSeek `deepseek-v4-flash`** during migration, and `null` otherwise. Callers treat `null` as "AI unavailable" and skip generation rather than throw. diff --git a/docs/data-layer.md b/docs/data-layer.md index e6aed748..4ea1b254 100644 --- a/docs/data-layer.md +++ b/docs/data-layer.md @@ -49,13 +49,14 @@ All three content tables share a common pattern: **Civic / elections** β€” the voter-information model: -| Table | Purpose | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `election` | Election records (external id, date, type, OCD division, deadlines JSONB) | -| `contest` | Races _and_ ballot measures. `type` = candidate \| referendum. For measures: `referendum_title`, pro/con statements, `summary`, `summary_is_ai_generated`, `fiscal_impact`, and a `citations` JSONB array (per-field source attribution: field, source name/url, trust tier, official flag) | -| `candidate` | Candidates within a contest (party, incumbent, contact, bio) | -| `polling_location` | Polling places / early-vote sites / drop boxes, geo-located (lat/long), with hours | -| `role_description` | Reusable descriptions of offices/roles by level (seeded with ~18 federalβ†’local roles) | +| Table | Purpose | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `election` | Election records (external id, date, type, OCD division, deadlines JSONB) | +| `contest` | Races _and_ ballot measures. `type` = candidate \| referendum. For measures: `referendum_title`, pro/con statements, `summary`, `summary_is_ai_generated`, `fiscal_impact`, and a `citations` JSONB array (per-field source attribution: field, source name/url, trust tier, official flag) | +| `candidate` | Candidates within a contest (party, incumbent, contact, bio) | +| `polling_location` | Polling places / early-vote sites / drop boxes, geo-located (lat/long), with hours | +| `role_description` | Reusable descriptions of offices/roles by level (seeded with ~18 federalβ†’local roles) | +| `election_source_snapshot` | Provider-neutral, idempotent current-cycle election snapshots with source version, checksum, diagnostics, and source URLs. Texas SOS facts and TLC analyses occupy separate rows. | **Local government (Legistar cache)** β€” `legistar_body`, `legistar_matter`, `legistar_meeting`, `legistar_agenda_item`, `legistar_vote`. These cache San Jose / Santa Clara / Sunnyvale council data (ordinances, meetings, agenda items, votes) keyed by `(jurisdiction, *_id)` with a `fetched_at` timestamp. diff --git a/docs/measure-enrichment.md b/docs/measure-enrichment.md index 950a9700..56efd1db 100644 --- a/docs/measure-enrichment.md +++ b/docs/measure-enrichment.md @@ -14,6 +14,7 @@ The Google Civic Information API reliably returns ballot-measure **titles** but ``` CA SOS Official Voter Guide ─┐ +TX SOS facts + TLC analysis ── LWV CaVotes (Pros & Cons) ── Ballotpedia (statewide+local)── Wikipedia (statewide props) ─┼──▢ Cross-Validation Engine ──▢ Canonical Measure ──▢ Cache (CivicApiCache) ──▢ App @@ -31,22 +32,23 @@ The engine fetches all sources concurrently (`Promise.all`) and merges them **fi When more than one source covers the same field, the higher tier wins (defined in `measure-sources/types.ts`, `SOURCE_TIER_RANK`): ``` -county_registrar > state_sos > lwv > ballotpedia > wikipedia > vote_smart > google_civic > ai_generated +county_registrar > state_sos = legislative_council > lwv > ballotpedia > wikipedia > vote_smart > google_civic > ai_generated ``` ## Source adapters (wired today) Each adapter fetches over a shared, defensive helper (`measure-sources/fetch.ts`) that uses a browser User-Agent and turns any failure into `null`. -| Source | Adapter | Tier | Scope / method | -| -------------------------------- | ---------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| CA SOS Official Voter Guide | `ca-sos-voterguide.ts` | `state_sos` | CA statewide props. Official AG summary, pro/con, full-text URL β€” **real official text, not AI.** Matches on the prop number parsed from the title. The guide is rebuilt each cycle and only serves the active election, so it yields nothing between prop cycles. | -| CA LAO Fiscal Analyses | `ca-lao-fiscal.ts` | `state_sos` | CA statewide props. Official nonpartisan fiscal impact analysis for each proposition. HTML scrape of `lao.ca.gov/BallotAnalysis/Proposition?number=N&year=YYYY` β€” no API exists. Pre-warmed by the `ca-lao-fiscal` scraper into `CivicApiCache`; adapter reads cache first, falls back to live fetch. Only fires for parsed proposition numbers (not local lettered measures). | -| League of Women Voters β€” CaVotes | `cavotes.ts` | `lwv` | CA statewide props. Nonpartisan "Pros & Cons" summary, fiscal effects, supporter/opponent arguments via the CaVotes WordPress REST API (`cavotes.org/wp-json/wp/v2/ballots`). Slugs are inconsistent across years, so it enumerates the list and matches by prop number + year. | -| Ballotpedia | `ballotpedia.ts` | `ballotpedia` | **Statewide _and_ local** lettered measures β€” the main source for local. Rendered article HTML (MediaWiki API disabled), resolved from a year/county index. Extracts ballot summary/question, fiscal impact / impartial analysis, arguments. Year-gated so a same-letter measure from another cycle isn't surfaced. | -| Wikipedia | `wikipedia.ts` | `wikipedia` | CA statewide props only β€” gated on a parsed prop number (local titles like "Measure Q" collide with unrelated articles). MediaWiki extract for ` California Proposition `; neutral encyclopedic overview. | -| Vote Smart | `votesmart.ts` | `vote_smart` | State-level measures. Fuzzy-matches the title to a Vote Smart measure β†’ summary, full-text URL, pro/con URLs. Requires `VOTE_SMART_API_KEY`. | -| Google Civic | (the input itself) | `google_civic` | The measure as the API returned it β€” lowest-trust, so its subtitle/text still surface when nothing better exists. | +| Source | Adapter | Tier | Scope / method | +| -------------------------------- | ---------------------- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| CA SOS Official Voter Guide | `ca-sos-voterguide.ts` | `state_sos` | CA statewide props. Official AG summary, pro/con, full-text URL β€” **real official text, not AI.** Matches on the prop number parsed from the title. The guide is rebuilt each cycle and only serves the active election, so it yields nothing between prop cycles. | +| CA LAO Fiscal Analyses | `ca-lao-fiscal.ts` | `state_sos` | CA statewide props. Official nonpartisan fiscal impact analysis for each proposition. HTML scrape of `lao.ca.gov/BallotAnalysis/Proposition?number=N&year=YYYY` β€” no API exists. Pre-warmed by the `ca-lao-fiscal` scraper into `CivicApiCache`; adapter reads cache first, falls back to live fetch. Only fires for parsed proposition numbers (not local lettered measures). | +| Texas SOS + Legislative Council | `texas-official.ts` | `state_sos` + `legislative_council` | Current-cycle Texas statewide propositions. Matches Google Civic/Vote Smart-style records by election date/year plus proposition number (title similarity fallback), then supplies separately cited SOS result facts and TLC page-cited explanation. | +| League of Women Voters β€” CaVotes | `cavotes.ts` | `lwv` | CA statewide props. Nonpartisan "Pros & Cons" summary, fiscal effects, supporter/opponent arguments via the CaVotes WordPress REST API (`cavotes.org/wp-json/wp/v2/ballots`). Slugs are inconsistent across years, so it enumerates the list and matches by prop number + year. | +| Ballotpedia | `ballotpedia.ts` | `ballotpedia` | **Statewide _and_ local** lettered measures β€” the main source for local. Rendered article HTML (MediaWiki API disabled), resolved from a year/county index. Extracts ballot summary/question, fiscal impact / impartial analysis, arguments. Year-gated so a same-letter measure from another cycle isn't surfaced. | +| Wikipedia | `wikipedia.ts` | `wikipedia` | CA statewide props only β€” gated on a parsed prop number (local titles like "Measure Q" collide with unrelated articles). MediaWiki extract for ` California Proposition `; neutral encyclopedic overview. | +| Vote Smart | `votesmart.ts` | `vote_smart` | State-level measures. Fuzzy-matches the title to a Vote Smart measure β†’ summary, full-text URL, pro/con URLs. Requires `VOTE_SMART_API_KEY`. | +| Google Civic | (the input itself) | `google_civic` | The measure as the API returned it β€” lowest-trust, so its subtitle/text still surface when nothing better exists. | **Local lettered measures:** the County Counsel / City Attorney **Impartial Analysis** (carried on Ballotpedia) is extracted on its own and wins the summary slot ahead of the bare ballot question β€” it's the authoritative neutral text, so it no longer gets buried in the fiscal field, and the advocacy/AI fallback only runs when it's absent. @@ -100,6 +102,7 @@ flowchart TD | `summaryIsAiGenerated` | true if `summary` came from AI | | `fiscalImpact` | official fiscal analysis | | `proArguments[]` / `conArguments[]` | attributed arguments (`{text, author?, sourceName, sourceUrl}`) | +| `result` | official SOS status, outcome, vote totals, choices, as-of time, and source link | | `citations[]` | which source supplied each field (`{field, sourceName, sourceUrl, tier, official}`) | | `sources[]` | one `Source` per citation, for attribution chips | diff --git a/docs/scraper.md b/docs/scraper.md index 68a83d34..cdbd91a6 100644 --- a/docs/scraper.md +++ b/docs/scraper.md @@ -15,21 +15,26 @@ process environment at runtime, not embedded during the build. ## Scrapers -| Scraper | Source | Content type | Method | -| ---------------------- | ------------------------------ | -------------------- | ------------------------------------------------------ | -| `congress.ts` | congress.gov REST API | `bill` | REST (`CONGRESS_API_KEY`), incremental by `updateDate` | -| `federalregister.ts` | federalregister.gov REST API | `government_content` | REST; HTMLβ†’Markdown via Turndown | -| `scotus.ts` | CourtListener REST API | `court_case` | REST (`COURTLISTENER_API_KEY`, optional) | -| `vote411.ts` | vote411.org | (cached locally) | cheerio HTML parse; does **not** write to the main DB | -| `scc-cvig.ts` | Santa Clara County voter guide | `civic_api_cache` | PDF extraction; optional Gemini fallback | -| `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | -| `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | -| `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | +| Scraper | Source | Content type | Method | +| --------------------------- | ------------------------------ | -------------------------- | ------------------------------------------------------ | +| `congress.ts` | congress.gov REST API | `bill` | REST (`CONGRESS_API_KEY`), incremental by `updateDate` | +| `federalregister.ts` | federalregister.gov REST API | `government_content` | REST; HTMLβ†’Markdown via Turndown | +| `scotus.ts` | CourtListener REST API | `court_case` | REST (`COURTLISTENER_API_KEY`, optional) | +| `vote411.ts` | vote411.org | (cached locally) | cheerio HTML parse; does **not** write to the main DB | +| `scc-cvig.ts` | Santa Clara County voter guide | `civic_api_cache` | PDF extraction; optional Gemini fallback | +| `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | +| `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | +| `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | +| `texas-current-election.ts` | Texas SOS + TLC | `election_source_snapshot` | current-cycle JSON + deterministic PDF text parsing | All HTTP goes through one `fetchWithRetry()` utility (`apps/scraper/src/utils/fetch.ts`): exponential backoff (1s/2s/4s…), `Retry-After` support (seconds or HTTP-date), 30s default timeout via `AbortController`, retriable on 429/5xx and `ECONNRESET`/`ECONNREFUSED`, plus a stateful **per-host backoff** that ramps on 429/5xx and relaxes on success. > Note: `whitehouse.gov` cheerio scraping was replaced by the structured **Federal Register** REST API. `vote411-ballot.ts` exists for address-based ballot lookup (needs Playwright) but isn't wired into the CLI. +The Texas scraper is intentionally current-cycle only. SOS election facts and +TLC explanatory text remain separate provider snapshots and separate citations; +see [Texas current-election data](./texas-current-election.md). + ## Upsert + Change Detection `apps/scraper/src/utils/db/operations.ts` centralizes writes behind a discriminated-union `upsertContent(type, data)` (`type` ∈ bill | government_content | court_case). Each run: diff --git a/docs/texas-current-election.md b/docs/texas-current-election.md new file mode 100644 index 00000000..5055cfed --- /dev/null +++ b/docs/texas-current-election.md @@ -0,0 +1,85 @@ +# Texas Current-Election Data + +## Scope + +`texas-current-election` ingests **only the current Texas election cycle** plus +the latest cycle-specific constitutional-amendment analysis. It does not expose +historical browsing, accept a year from API callers, or backfill older cycles. + +The Texas Secretary of State (SOS) is authoritative for statewide election +facts: elections, contests, candidates, vote totals, county totals, turnout, +reporting progress, outcomes, and whether the results application labels an +election official. The Texas Legislative Council (TLC) supplies nonpartisan +explanatory text: ballot language, summary/background analysis, supporter and +opponent comments, and fiscal implications. These providers are persisted in +separate rows and receive separate field-level citations in the reader. + +Texas SOS is **not a complete source for local Texas elections**. This pipeline +normalizes the statewide, federal, and district records the SOS publishes; it +does not imply that every city, school-district, county, or special-district +contest is present. + +## Sources and discovery + +- SOS results app: `https://goelect.txelections.civixapps.com/ivis-enr-ui/races` +- TLC publications: `https://tlc.texas.gov/publications` + +The scraper reads the SOS application's `electionConstants` JSON to discover +the newest cycle and its election IDs. It never constructs a URL from a +hard-coded year. It also discovers the newest non-condensed `analysesNN.pdf` +link from the TLC publications HTML. When the TLC report belongs to the prior +calendar year (for example, the 2025 amendment election during the 2026 general +cycle), the scraper fetches only that matching SOS constitutional-amendment +election so the current TLC analysis can be paired with its SOS outcome. + +The Civix API wraps sections as base64 JSON. Parsing is deterministic and +normalizes: + +- candidate and proposition contests, parties, incumbency, votes, percentages, + early votes, winners, and proposition outcomes; +- counties/precincts/polling places reporting and official/unofficial status; +- per-county choice totals and turnout; +- source version and SHA-256 content checksum. + +TLC PDFs are read from their text layer page-by-page. There is no AI or vision +step in the normal path. Each extracted field links to `#page=N` in the official +PDF. Missing optional sections produce diagnostics and do not discard the rest +of the proposition. + +## Persistence and reader + +The provider-neutral `election_source_snapshot` table stores one idempotent row +per `(jurisdiction, cycle_year, provider, scope)`. Texas writes `texas-sos` and +`texas-tlc` rows with `scope = current`; certification/status refreshes update +those rows instead of creating duplicates. + +Apply `packages/db/migrations/add_election_source_snapshots.sql` before the +first run, then execute: + +```sh +pnpm --filter @acme/scraper run start texas-current-election +``` + +`TX_SOS_MAX_ITEMS` (default `12`) or the CLI `--max-items` flag caps source +records per run. `POSTGRES_URL` is the only required credential; both sources +are public and require no API key. + +Consumers call the public tRPC query `civic.getTexasCurrentElection`. It has no +input by design. The response contains the newest SOS cycle and the latest TLC +amendment cycle. TLC propositions are matched to SOS outcomes by election cycle +and proposition number; each result retains its SOS citation while every +analysis field retains its TLC page citation. + +The same persisted data is wired into `civic.getVoterInfo` through +`measure-sources/texas-official.ts`. A Google Civic measure is matched by exact +election date/year and proposition number, with title similarity only as a +fallback when the number is absent. Vote Smart matching also rejects records +whose published election date differs. The canonical measure exposes SOS +`result` facts and TLC summary/fiscal/pro-con fields with separate citations. + +## Verification + +Fixtures cover candidate results, county amendment totals, the 2025 TLC layout, +and the alternate 2023 layout. Tests assert current-cycle discovery, idempotent +identities, official status, turnout/outcomes, page citations, source separation, +and fail-soft behavior for a missing section. diff --git a/packages/api/package.json b/packages/api/package.json index 661eb542..7b46d5ad 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -47,6 +47,10 @@ "types": "./dist/lib/measure-sources/types.d.ts", "default": "./src/lib/measure-sources/types.ts" }, + "./lib/texas-election-data": { + "types": "./dist/lib/texas-election-data.d.ts", + "default": "./src/lib/texas-election-data.ts" + }, "./lib/measure-sources/disabled/vig-archive": { "types": "./dist/lib/measure-sources/disabled/vig-archive.d.ts", "default": "./src/lib/measure-sources/disabled/vig-archive.ts" diff --git a/packages/api/src/lib/__fixtures__/texas-sos-amendment.json b/packages/api/src/lib/__fixtures__/texas-sos-amendment.json new file mode 100644 index 00000000..e730f3d3 --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-sos-amendment.json @@ -0,0 +1,5 @@ +{ + "Version": "enr/51031/86/", + "Home": "eyJFbGVjRGF0ZSI6IjExMDQyMDI1IiwiQ291bnRpZXNSZXBvcnRpbmciOnsiQ1IiOjI1NCwiQ1QiOjI1NH0sIkxhc3RVcGRhdGVkVGltZSI6Ik5vdiAwNiwgMjAyNSAwNDoxODoyMyIsIlByZWNpbmN0c1JlcG9ydGluZyI6eyJQUiI6MTE3MiwiUFQiOjExNzJ9LCJQb2xsaW5nUmVwb3J0aW5nIjp7IlBMUiI6NDYzNywiUExUIjo0NjM3fX0=", + "Race": "eyJPZmZpY2VUeXBlcyI6W3siT2ZmaWNlVHlwZSI6IlNUQVRFV0lERSBQUk9QT1NJVElPTlMiLCJSYWNlcyI6W3siaWQiOjg3LCJOIjoiUFJPUE9TSVRJT04gMSAoU0pSIDU5KSIsIkNhbmRpZGF0ZXMiOlt7Ik4iOiJGT1IiLCJWIjoyMDQxODU5LCJQRSI6NjkuMDMsIkVWIjoxMDA4MzkxfSx7Ik4iOiJBR0FJTlNUIiwiViI6OTE2MjE3LCJQRSI6MzAuOTcsIkVWIjo0NTY1Mzd9XSwiVCI6Mjk1ODA3Nn1dfV19" +} diff --git a/packages/api/src/lib/__fixtures__/texas-sos-constants.json b/packages/api/src/lib/__fixtures__/texas-sos-constants.json new file mode 100644 index 00000000..a42bddc9 --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-sos-constants.json @@ -0,0 +1,3 @@ +{ + "upload": "eyJlbGVjdGlvbkluZm8iOnsiMjAyNiI6eyJQIjp7IjUzODEzIjp7IklEIjo1MzgxMywiTiI6IjIwMjYgUkVQVUJMSUNBTiBQUklNQVJZIEVMRUNUSU9OIiwiTyI6IlkiLCJZIjoiMjAyNiIsIkVUIjoiUCIsIkVDIjoiU1cifX0sIkdFIjp7IjUzODE1Ijp7IklEIjo1MzgxNSwiTiI6IjIwMjYgTk9WRU1CRVIgR0VORVJBTCBFTEVDVElPTiIsIk8iOiJOIiwiWSI6IjIwMjYiLCJFVCI6IkdFIiwiRUMiOiJTVyJ9fX0sIjIwMjUiOnsiUyI6eyI1MTAzMSI6eyJJRCI6NTEwMzEsIk4iOiIyMDI1IE5PVkVNQkVSIDRUSCBDT05TVElUVVRJT05BTCBBTUVORE1FTlQiLCJPIjoiTiIsIlkiOiIyMDI1IiwiRVQiOiJTIiwiRUMiOiJTVyJ9fX19fQ==" +} diff --git a/packages/api/src/lib/__fixtures__/texas-sos-county.json b/packages/api/src/lib/__fixtures__/texas-sos-county.json new file mode 100644 index 00000000..e6d5e5fb --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-sos-county.json @@ -0,0 +1,3 @@ +{ + "upload": "eyIxIjp7Ik4iOiJBTkRFUlNPTiIsIlRWIjo4MDU0NCwiUmFjZXMiOnsiODciOnsiT0lEIjo4NywiTiI6IlBST1BPU0lUSU9OIDEgKFNKUiA1OSkiLCJDIjp7Ijg3MDEiOnsiaWQiOjg3MDEsIk4iOiJGT1IiLCJWIjozMzYxLCJQRSI6NzAuOTIsIkVWIjoxNTAyfSwiODcwMiI6eyJpZCI6ODcwMiwiTiI6IkFHQUlOU1QiLCJWIjoxMzc4LCJQRSI6MjkuMDgsIkVWIjo2MDl9fSwiUFIiOjIzLCJUUCI6MjN9fSwiU3VtbWFyeSI6eyJSViI6MzA1NzksIlZDIjo0NzM5LCJWVCI6MTUuNX19fQ==" +} diff --git a/packages/api/src/lib/__fixtures__/texas-sos-election.json b/packages/api/src/lib/__fixtures__/texas-sos-election.json new file mode 100644 index 00000000..af41c508 --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-sos-election.json @@ -0,0 +1,5 @@ +{ + "Version": "enr/53813/135/", + "Home": "eyJFbGVjRGF0ZSI6IjAzMDMyMDI2IiwiQ291bnRpZXNSZXBvcnRpbmciOnsiQ1IiOjI1NCwiQ1QiOjI1NH0sIkxhc3RVcGRhdGVkVGltZSI6IkFwciAxNCwgMjAyNiAyMDo0ODoxNyIsIlByZWNpbmN0c1JlcG9ydGluZyI6eyJQUiI6NSwiUFQiOjV9LCJQb2xsaW5nUmVwb3J0aW5nIjp7IlBMUiI6NDYzNiwiUExUIjo0NjM2fX0=", + "Race": "eyJPZmZpY2VUeXBlcyI6W3siT2ZmaWNlVHlwZSI6IlNUQVRFV0lERSBPRkZJQ0VTIiwiUmFjZXMiOlt7ImlkIjozNTc5LCJOIjoiR09WRVJOT1IiLCJDYW5kaWRhdGVzIjpbeyJJRCI6MjUyNzMsIk4iOiJHUkVHIEFCQk9UVCAoSSkiLCJQIjoiUkVQIiwiViI6MTc2NDkyNCwiUEUiOjgxLjgzLCJFViI6MTEzMzEyNX0seyJJRCI6MzI4MTUsIk4iOiJSLkYuIFwiQk9CXCIgQUNIR0lMTCIsIlAiOiJSRVAiLCJWIjo5MTE5LCJQRSI6MC40MiwiRVYiOjU2MTd9XSwiVCI6MTc3NDA0M31dfV19" +} diff --git a/packages/api/src/lib/__fixtures__/texas-tlc-2023.json b/packages/api/src/lib/__fixtures__/texas-tlc-2023.json new file mode 100644 index 00000000..8366dfd2 --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-tlc-2023.json @@ -0,0 +1,4 @@ +[ + {"page":1,"text":"Analyses of Proposed Constitutional Amendments November 7, 2023, Election Texas Legislative Council"}, + {"page":9,"text":"PROPOSITION 1 (H.J.R. 126) The constitutional amendment protecting the right to engage in farming. SUMMARY ANALYSIS The proposal protects generally accepted agricultural practices. BACKGROUND Existing statutes regulate agricultural operations. SUMMARY OF COMMENTS Comments by Supporters: β€’ The amendment would protect farms from unnecessary regulation. Text of H.J.R. 126"} +] diff --git a/packages/api/src/lib/__fixtures__/texas-tlc-2025.json b/packages/api/src/lib/__fixtures__/texas-tlc-2025.json new file mode 100644 index 00000000..c74af3e3 --- /dev/null +++ b/packages/api/src/lib/__fixtures__/texas-tlc-2025.json @@ -0,0 +1,7 @@ +[ + {"page":1,"text":"ANALYSES OF PROPOSED CONSTITUTIONAL AMENDMENTS 89TH TEXAS LEGISLATURE NOVEMBER 4, 2025, ELECTION Published by the Texas Legislative Council August 2025"}, + {"page":10,"text":"Proposition 1 (S.J.R. 59) The constitutional amendment providing for the creation of the permanent technical institution infrastructure fund. SUMMARY ANALYSIS S.J.R. 59 proposes an amendment creating two funds for technical education. The legislature appropriated $850 million to the permanent fund. BACKGROUND AND DETAILED ANALYSIS Existing law provides general funding to public institutions. The new fund would provide a dedicated source of revenue."}, + {"page":11,"text":"SUMMARY OF COMMENTS Comments by Supporters: β€’ The amendment would expand workforce training capacity. β€’ A dedicated fund would improve long-term planning. Comments by Opponents: β€’ A perpetual fund would reduce the discretion of future legislatures."}, + {"page":12,"text":"Text of S.J.R. 59 SENATE JOINT RESOLUTION proposing a constitutional amendment."}, + {"page":14,"text":"Proposition 2 (S.J.R. 18) The constitutional amendment prohibiting a tax on capital gains. SUMMARY ANALYSIS The resolution prohibits a tax. BACKGROUND AND DETAILED ANALYSIS Texas does not currently impose this tax. SUMMARY OF COMMENTS Comments by Supporters: β€’ The prohibition gives taxpayers certainty. Comments by Opponents: β€’ The constitution should not contain tax policy. Text of S.J.R. 18"} +] diff --git a/packages/api/src/lib/civic.ts b/packages/api/src/lib/civic.ts index b268ea16..c39b2199 100644 --- a/packages/api/src/lib/civic.ts +++ b/packages/api/src/lib/civic.ts @@ -16,6 +16,7 @@ import type { StatewideOffice, } from "../clients/ca-sos-results"; import type { CrossValidateContext } from "./measure-crossvalidate"; +import type { MeasureResult } from "./measure-sources/types"; import { getDistrictResults, getStatewideResults, @@ -217,6 +218,8 @@ export interface Contest { conArguments?: MeasureArgumentRef[]; /** Per-field source attribution for everything above. */ citations?: MeasureCitationRef[]; + /** Official election result facts when an SOS match is available. */ + result?: MeasureResult; } export interface Candidate { @@ -472,6 +475,7 @@ interface EnrichmentContext { stateAbbrev?: string; county?: string; electionYear?: number; + electionDate?: string; } /** @@ -514,6 +518,7 @@ async function enrichContest( stateAbbrev: ctx?.stateAbbrev, county: ctx?.county, electionYear: ctx?.electionYear ?? new Date().getFullYear(), + electionDate: ctx?.electionDate, }; try { @@ -543,6 +548,7 @@ async function enrichContest( contest.citations = merged.citations.length ? merged.citations : undefined; + contest.result = merged.result; // Back-fill the legacy single-field shape so existing UI keeps working. if (!contest.referendumText && merged.fullText) { @@ -838,6 +844,7 @@ export async function getVoterInfo( electionYear: resp.election.electionDay ? new Date(resp.election.electionDay).getFullYear() : new Date().getFullYear(), + electionDate: resp.election.electionDay, }); const params: Record = { address }; diff --git a/packages/api/src/lib/measure-crossvalidate.ts b/packages/api/src/lib/measure-crossvalidate.ts index 1fd95ece..15a5e63d 100644 --- a/packages/api/src/lib/measure-crossvalidate.ts +++ b/packages/api/src/lib/measure-crossvalidate.ts @@ -23,6 +23,7 @@ import { enrichFromLao } from "./measure-sources/ca-lao-fiscal"; import { enrichFromCaSos } from "./measure-sources/ca-sos-voterguide"; import { enrichFromCaVotes } from "./measure-sources/cavotes"; import { collectGroundingText } from "./measure-sources/grounded-fallback"; +import { enrichFromTexasOfficial } from "./measure-sources/texas-official"; import { SOURCE_TIER_RANK } from "./measure-sources/types"; import { enrichFromWikipedia } from "./measure-sources/wikipedia"; import { enrichFromVoteSmart } from "./votesmart"; @@ -31,6 +32,7 @@ export interface CrossValidateContext { stateAbbrev?: string; county?: string; electionYear: number; + electionDate?: string; } /** A measure as Google Civic gives it to us β€” the lowest-tier source. */ @@ -55,6 +57,7 @@ async function collectVoteSmart( title, ctx.stateAbbrev, ctx.electionYear, + ctx.electionDate, ).catch(() => null); if (!vs) return null; return { @@ -142,7 +145,7 @@ export async function crossValidateMeasure( input: CivicMeasureInput, ctx: CrossValidateContext, ): Promise { - const [sos, lwv, bp, wiki, vs, lao] = await Promise.all([ + const [sos, lwv, bp, wiki, vs, lao, texas] = await Promise.all([ enrichFromCaSos(input.title, ctx.stateAbbrev, ctx.electionYear).catch( () => null, ), @@ -161,6 +164,9 @@ export async function crossValidateMeasure( enrichFromLao(input.title, ctx.stateAbbrev, ctx.electionYear).catch( () => null, ), + enrichFromTexasOfficial(input.title, ctx).catch( + (): { sos?: MeasureSourceData; tlc?: MeasureSourceData } => ({}), + ), ]); const sources: MeasureSourceData[] = [ @@ -170,6 +176,8 @@ export async function crossValidateMeasure( wiki, vs, lao, + texas.sos ?? null, + texas.tlc ?? null, civicAsSource(input), ].filter((s): s is MeasureSourceData => s !== null); sources.sort(byTierDesc); @@ -217,6 +225,11 @@ export async function crossValidateMeasure( } } + // --- Official result facts: SOS only; explanatory sources leave absent. --- + const resultSource = sources.find((source) => source.result); + const result = resultSource?.result; + if (resultSource && result) citations.push(cite("result", resultSource)); + // --- Pro / con arguments: collect from all sources, dedupe, attribute. --- const proArguments = collectArguments(sources, "pro", citations); const conArguments = collectArguments(sources, "con", citations); @@ -381,6 +394,7 @@ export async function crossValidateMeasure( fullTextUrl, proArguments, conArguments, + result, citations, discrepancies: discrepancies.length ? discrepancies : undefined, }; diff --git a/packages/api/src/lib/measure-sources/texas-official.test.ts b/packages/api/src/lib/measure-sources/texas-official.test.ts new file mode 100644 index 00000000..7f177eb1 --- /dev/null +++ b/packages/api/src/lib/measure-sources/texas-official.test.ts @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import type { TexasCurrentElectionData } from "../texas-election-data"; +import { matchTexasOfficialMeasure } from "./texas-official"; + +const current: TexasCurrentElectionData = { + jurisdiction: "TX", + cycleYear: 2026, + elections: [], + fetchedAt: "2026-07-21T00:00:00.000Z", + diagnostics: [], + constitutionalAmendments: { + cycleYear: 2025, + electionDate: "November 4, 2025", + citation: { + sourceName: "Texas Legislative Council", + sourceUrl: "https://tlc.texas.gov/docs/amendments/analyses25.pdf", + provider: "texas-tlc", + official: true, + }, + measures: [ + { + propositionNumber: 1, + ballotLanguage: + "The constitutional amendment creating a permanent technical institution infrastructure fund.", + summaryAnalysis: "Creates a dedicated education infrastructure fund.", + supporterArguments: ["Supporters say the fund expands training."], + opponentArguments: ["Opponents question a permanent appropriation."], + fiscalImplications: ["The legislature appropriated $850 million."], + pageStart: 10, + pageEnd: 17, + diagnostics: [], + citations: { + summaryAnalysis: { + sourceName: "Texas Legislative Council", + sourceUrl: + "https://tlc.texas.gov/docs/amendments/analyses25.pdf#page=10", + provider: "texas-tlc", + official: true, + page: 10, + }, + }, + result: { + status: "complete", + outcome: "adopted", + totalVotes: 100, + choices: [ + { + name: "FOR", + incumbent: false, + votes: 70, + percent: 70, + winner: true, + }, + { + name: "AGAINST", + incumbent: false, + votes: 30, + percent: 30, + winner: false, + }, + ], + citation: { + sourceName: "Texas Secretary of State", + sourceUrl: + "https://goelect.txelections.civixapps.com/ivis-enr-ui/races", + provider: "texas-sos", + official: true, + }, + }, + }, + ], + }, +}; + +void test("matches Google/Vote Smart-style titles by date and proposition number", () => { + const matched = matchTexasOfficialMeasure( + "Texas Proposition 1, Technical Education Fund Amendment", + { + stateAbbrev: "TX", + electionYear: 2025, + electionDate: "2025-11-04", + }, + current, + ); + const amendment = current.constitutionalAmendments; + assert.ok(amendment); + assert.ok(matched.sos); + assert.ok(matched.tlc); + assert.equal(matched.sos.result?.outcome, "adopted"); + assert.equal( + matched.tlc.officialSummary, + amendment.measures[0]?.summaryAnalysis, + ); + assert.equal(matched.sos.sourceName, "Texas Secretary of State"); + assert.equal(matched.tlc.sourceName, "Texas Legislative Council"); +}); + +void test("rejects a proposition from a different election date", () => { + const matched = matchTexasOfficialMeasure( + "Proposition 1", + { + stateAbbrev: "TX", + electionYear: 2025, + electionDate: "2025-05-03", + }, + current, + ); + assert.deepEqual(matched, {}); +}); diff --git a/packages/api/src/lib/measure-sources/texas-official.ts b/packages/api/src/lib/measure-sources/texas-official.ts new file mode 100644 index 00000000..7b190bda --- /dev/null +++ b/packages/api/src/lib/measure-sources/texas-official.ts @@ -0,0 +1,113 @@ +/** Current-cycle Texas SOS facts and TLC explanations for Google Civic measures. */ + +import type { TexasCurrentElectionData } from "../texas-election-data"; +import type { MeasureSourceData } from "./types"; +import { getTexasCurrentElectionData } from "../texas-election-data"; +import { titleSimilarity } from "../votesmart"; + +function propositionNumber(title: string): number | undefined { + const value = /\b(?:proposition|prop\.?)\s*(\d+)\b/i.exec(title)?.[1]; + return value ? Number.parseInt(value, 10) : undefined; +} + +function sameDate(a: string | undefined, b: string | undefined): boolean { + if (!a || !b) return true; + const left = new Date(a); + const right = new Date(b); + if (Number.isNaN(left.getTime()) || Number.isNaN(right.getTime())) { + return a.toLowerCase() === b.toLowerCase(); + } + return left.toISOString().slice(0, 10) === right.toISOString().slice(0, 10); +} + +/** + * Match by election date/year plus proposition number, falling back to a title + * similarity check only when Google omits the number. The returned SOS and TLC + * source objects stay separate so field citations cannot collapse them. + */ +export async function enrichFromTexasOfficial( + title: string, + context: { + stateAbbrev?: string; + electionYear: number; + electionDate?: string; + }, +): Promise<{ sos?: MeasureSourceData; tlc?: MeasureSourceData }> { + if (context.stateAbbrev?.toUpperCase() !== "TX") return {}; + const current = await getTexasCurrentElectionData().catch(() => null); + return matchTexasOfficialMeasure(title, context, current); +} + +/** Pure matcher used by the reader and fixture tests. */ +export function matchTexasOfficialMeasure( + title: string, + context: { + stateAbbrev?: string; + electionYear: number; + electionDate?: string; + }, + current: TexasCurrentElectionData | null, +): { sos?: MeasureSourceData; tlc?: MeasureSourceData } { + if (context.stateAbbrev?.toUpperCase() !== "TX") return {}; + const amendments = current?.constitutionalAmendments; + if ( + amendments?.cycleYear !== context.electionYear || + !sameDate(context.electionDate, amendments.electionDate) + ) { + return {}; + } + + const number = propositionNumber(title); + const measure = amendments.measures.find((candidate) => { + if (number !== undefined) return candidate.propositionNumber === number; + return titleSimilarity(title, candidate.ballotLanguage ?? "") >= 0.45; + }); + if (!measure) return {}; + + const tlc: MeasureSourceData = { + tier: "legislative_council", + sourceName: "Texas Legislative Council", + sourceUrl: measure.citations.summaryAnalysis?.sourceUrl, + official: true, + matchedTitle: measure.ballotLanguage, + officialSummary: measure.summaryAnalysis, + fiscalImpact: measure.fiscalImplications.join(" ") || undefined, + fullText: measure.ballotLanguage, + fullTextUrl: measure.citations.ballotLanguage?.sourceUrl, + proArguments: measure.supporterArguments.map((text) => ({ + text, + sourceName: "Texas Legislative Council", + sourceUrl: measure.citations.supporterArguments?.sourceUrl, + })), + conArguments: measure.opponentArguments.map((text) => ({ + text, + sourceName: "Texas Legislative Council", + sourceUrl: measure.citations.opponentArguments?.sourceUrl, + })), + }; + + const result = measure.result; + const sos: MeasureSourceData | undefined = result + ? { + tier: "state_sos", + sourceName: "Texas Secretary of State", + sourceUrl: result.citation.sourceUrl, + official: true, + matchedTitle: title, + result: { + status: result.status, + outcome: result.outcome, + totalVotes: result.totalVotes, + choices: result.choices.map((choice) => ({ + name: choice.name, + votes: choice.votes, + percent: choice.percent, + })), + sourceName: "Texas Secretary of State", + sourceUrl: result.citation.sourceUrl, + asOf: result.asOf, + }, + } + : undefined; + return { sos, tlc }; +} diff --git a/packages/api/src/lib/measure-sources/types.ts b/packages/api/src/lib/measure-sources/types.ts index d39683d0..6cb6acee 100644 --- a/packages/api/src/lib/measure-sources/types.ts +++ b/packages/api/src/lib/measure-sources/types.ts @@ -11,13 +11,14 @@ * Trust tiers, highest to lowest. The cross-validation engine prefers data * from a higher tier when multiple sources cover the same field. * - * County registrar / State SOS (official) > League of Women Voters + * County registrar / State SOS / legislative council (official) > League of Women Voters * (nonpartisan) > Ballotpedia (aggregator) > Wikipedia (encyclopedic) > * Vote Smart > Google Civic > AI grounded on fetched text (last resort). */ export type SourceTier = | "county_registrar" | "state_sos" + | "legislative_council" | "lwv" | "ballotpedia" | "wikipedia" @@ -28,6 +29,7 @@ export type SourceTier = export const SOURCE_TIER_RANK: Record = { county_registrar: 8, state_sos: 7, + legislative_council: 7, lwv: 6, ballotpedia: 5, wikipedia: 4, @@ -77,6 +79,18 @@ export interface MeasureSourceData { /** Single pro/con statement (Google Civic shape) when no list is available. */ proStatement?: string; conStatement?: string; + /** Official election result facts; explanatory sources leave this absent. */ + result?: MeasureResult; +} + +export interface MeasureResult { + status: "upcoming" | "reporting" | "complete" | "official"; + outcome?: "adopted" | "rejected"; + totalVotes: number; + choices: { name: string; votes: number; percent: number }[]; + asOf?: string; + sourceName: string; + sourceUrl: string; } /** @@ -97,6 +111,7 @@ export interface CanonicalMeasure { fullTextUrl?: string; proArguments: MeasureArgument[]; conArguments: MeasureArgument[]; + result?: MeasureResult; citations: MeasureCitation[]; /** Fields where sources disagreed, flagged for human review. */ discrepancies?: string[]; diff --git a/packages/api/src/lib/texas-election-data.test.ts b/packages/api/src/lib/texas-election-data.test.ts new file mode 100644 index 00000000..e47b22fa --- /dev/null +++ b/packages/api/src/lib/texas-election-data.test.ts @@ -0,0 +1,113 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +import { + parseTexasSosDiscovery, + parseTexasSosElection, + parseTexasTlcAnalysis, + TEXAS_SOS_PROVIDER, + TEXAS_TLC_PROVIDER, +} from "./texas-election-data"; + +function fixture(name: string): T { + return JSON.parse( + readFileSync(new URL(`./__fixtures__/${name}`, import.meta.url), "utf8"), + ) as T; +} + +void test("SOS discovery selects the current cycle without hard-coded year URLs", () => { + const discovery = parseTexasSosDiscovery(fixture("texas-sos-constants.json")); + assert.equal(discovery.cycleYear, 2026); + assert.deepEqual( + discovery.elections.map((election) => election.id), + [53813, 53815], + ); + assert.equal(discovery.electionsByYear[2025]?.[0]?.id, 51031); +}); + +void test("SOS parser normalizes current candidate contests and official status", () => { + const discovery = parseTexasSosDiscovery(fixture("texas-sos-constants.json")); + const definition = discovery.elections.find( + (election) => election.id === 53813, + ); + assert.ok(definition); + const election = parseTexasSosElection( + fixture("texas-sos-election.json"), + definition, + undefined, + new Date("2026-07-21T00:00:00Z"), + ); + assert.equal(election.status, "official"); + assert.equal(election.reporting.percentReporting, 100); + const contest = election.contests[0]; + assert.ok(contest); + const choice = contest.choices[0]; + assert.ok(choice); + assert.equal(contest.title, "GOVERNOR"); + assert.equal(choice.name, "GREG ABBOTT"); + assert.equal(choice.incumbent, true); + assert.equal(contest.citation.provider, TEXAS_SOS_PROVIDER); +}); + +void test("SOS amendment results include outcome, county totals, and turnout", () => { + const discovery = parseTexasSosDiscovery(fixture("texas-sos-constants.json")); + const definition = discovery.electionsByYear[2025]?.[0]; + assert.ok(definition); + const election = parseTexasSosElection( + fixture("texas-sos-amendment.json"), + definition, + fixture("texas-sos-county.json"), + new Date("2026-07-21T00:00:00Z"), + ); + const contest = election.contests[0]; + assert.ok(contest); + assert.equal(election.status, "complete"); + assert.equal(contest.type, "referendum"); + assert.equal(contest.propositionNumber, 1); + assert.equal(contest.outcome, "adopted"); + const county = contest.counties[0]; + assert.ok(county); + const countyChoice = county.choices[0]; + assert.ok(countyChoice); + assert.equal(county.county, "ANDERSON"); + assert.equal(countyChoice.votes, 3361); + assert.ok(election.turnout); + assert.equal(election.turnout.registeredVoters, 80544); + assert.equal(election.turnout.ballotsCast, 4739); +}); + +void test("TLC 2025 layout preserves page citations and separate explanation tier", () => { + const parsed = parseTexasTlcAnalysis( + fixture("texas-tlc-2025.json"), + "https://tlc.texas.gov/docs/amendments/analyses25.pdf", + ); + assert.equal(parsed.cycleYear, 2025); + assert.equal(parsed.electionDate, "NOVEMBER 4, 2025"); + assert.equal(parsed.measures.length, 2); + const first = parsed.measures[0]; + assert.ok(first); + assert.equal(first.resolution, "SJR 59"); + assert.match(first.ballotLanguage ?? "", /permanent technical institution/i); + assert.equal(first.supporterArguments.length, 2); + assert.equal(first.opponentArguments.length, 1); + assert.match(first.fiscalImplications[0] ?? "", /\$850 million/); + const summaryCitation = first.citations.summaryAnalysis; + assert.ok(summaryCitation); + assert.equal(summaryCitation.provider, TEXAS_TLC_PROVIDER); + assert.equal(summaryCitation.page, 10); + assert.match(summaryCitation.sourceUrl, /#page=10$/); +}); + +void test("TLC 2023 alternate layout fails soft when a section is absent", () => { + const parsed = parseTexasTlcAnalysis( + fixture("texas-tlc-2023.json"), + "https://tlc.texas.gov/docs/amendments/analyses23.pdf", + ); + const measure = parsed.measures[0]; + assert.ok(measure); + assert.equal(measure.propositionNumber, 1); + assert.match(measure.background ?? "", /Existing statutes/); + assert.deepEqual(measure.opponentArguments, []); + assert.ok(measure.diagnostics.includes("missing opponent comments")); +}); diff --git a/packages/api/src/lib/texas-election-data.ts b/packages/api/src/lib/texas-election-data.ts new file mode 100644 index 00000000..adb1865d --- /dev/null +++ b/packages/api/src/lib/texas-election-data.ts @@ -0,0 +1,808 @@ +/** + * Deterministic parsers and reader types for the Texas SOS/TLC current-cycle + * election handoff. SOS result facts and TLC explanatory text deliberately use + * distinct records and citations all the way to the API response. + */ + +import { and, desc, eq } from "@acme/db"; +import { db } from "@acme/db/client"; +import { ElectionSourceSnapshot } from "@acme/db/schema"; + +export const TEXAS_SOS_PROVIDER = "texas-sos"; +export const TEXAS_TLC_PROVIDER = "texas-tlc"; +export const TEXAS_CURRENT_SCOPE = "current"; +export const TEXAS_RESULTS_URL = + "https://goelect.txelections.civixapps.com/ivis-enr-ui/races"; +export const TEXAS_TLC_PUBLICATIONS_URL = "https://tlc.texas.gov/publications"; + +export interface SourceCitation { + sourceName: string; + sourceUrl: string; + official: true; + provider: typeof TEXAS_SOS_PROVIDER | typeof TEXAS_TLC_PROVIDER; + page?: number; +} + +export interface TexasElectionDefinition { + id: number; + name: string; + year: number; + type: string; + category?: string; + official: boolean; +} + +export interface TexasSosDiscovery { + cycleYear: number; + elections: TexasElectionDefinition[]; + electionsByYear: Record; +} + +export interface TexasResultChoice { + id?: number; + name: string; + party?: string; + incumbent: boolean; + votes: number; + earlyVotes?: number; + percent: number; + winner: boolean; +} + +export interface TexasCountyResult { + countyId: number; + county: string; + registeredVoters?: number; + ballotsCast?: number; + turnoutPercent?: number; + reportingPercent?: number; + choices: TexasResultChoice[]; +} + +export interface TexasElectionContest { + id: string; + sourceContestId: number; + type: "candidate" | "referendum"; + title: string; + officeType?: string; + propositionNumber?: number; + resolution?: string; + totalVotes: number; + choices: TexasResultChoice[]; + counties: TexasCountyResult[]; + outcome?: "adopted" | "rejected"; + citation: SourceCitation; +} + +export interface TexasElectionReporting { + countiesReporting?: number; + countiesTotal?: number; + precinctsReporting?: number; + precinctsTotal?: number; + pollingPlacesReporting?: number; + pollingPlacesTotal?: number; + percentReporting?: number; +} + +export interface TexasElectionTurnout { + registeredVoters?: number; + ballotsCast?: number; + turnoutPercent?: number; +} + +export interface TexasSosElection { + id: number; + name: string; + date?: string; + year: number; + electionType: string; + status: "upcoming" | "reporting" | "complete" | "official"; + official: boolean; + lastUpdated?: string; + sourceVersion: string; + reporting: TexasElectionReporting; + turnout?: TexasElectionTurnout; + contests: TexasElectionContest[]; + citation: SourceCitation; +} + +export interface TexasSosSnapshotData { + cycleYear: number; + elections: TexasSosElection[]; +} + +export interface TlcTextPage { + page: number; + text: string; +} + +export interface TexasTlcMeasure { + propositionNumber: number; + resolution?: string; + ballotLanguage?: string; + summaryAnalysis?: string; + background?: string; + supporterArguments: string[]; + opponentArguments: string[]; + fiscalImplications: string[]; + pageStart: number; + pageEnd: number; + citations: { + ballotLanguage?: SourceCitation; + summaryAnalysis?: SourceCitation; + background?: SourceCitation; + supporterArguments?: SourceCitation; + opponentArguments?: SourceCitation; + fiscalImplications?: SourceCitation; + }; + diagnostics: string[]; +} + +export interface TexasTlcSnapshotData { + cycleYear: number; + electionDate?: string; + publicationTitle?: string; + sourceUrl: string; + measures: TexasTlcMeasure[]; +} + +export interface TexasCurrentElectionData { + jurisdiction: "TX"; + cycleYear: number; + elections: TexasSosElection[]; + constitutionalAmendments?: { + cycleYear: number; + electionDate?: string; + publicationTitle?: string; + measures: (TexasTlcMeasure & { + result?: Pick< + TexasElectionContest, + "outcome" | "totalVotes" | "choices" | "citation" + > & { + status: TexasSosElection["status"]; + asOf?: string; + }; + })[]; + citation: SourceCitation; + }; + diagnostics: string[]; + fetchedAt: string; +} + +type JsonObject = Record; + +function object(value: unknown): JsonObject | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as JsonObject) + : null; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function numberValue(value: unknown): number | undefined { + const parsed = typeof value === "number" ? value : Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function decodeJson(value: unknown): unknown { + if (typeof value !== "string" || !value) return null; + try { + return JSON.parse(Buffer.from(value, "base64").toString("utf8")) as unknown; + } catch { + return null; + } +} + +function unwrapUpload(value: unknown): unknown { + const envelope = object(value); + return envelope && "upload" in envelope ? decodeJson(envelope.upload) : value; +} + +/** Parse the SOS discovery envelope and return only structured definitions. */ +export function parseTexasSosDiscovery(raw: unknown): TexasSosDiscovery { + const decoded = object(unwrapUpload(raw)); + const years = object(decoded?.electionInfo); + if (!years) + throw new Error("Texas SOS discovery payload has no electionInfo"); + + const electionsByYear: Record = {}; + for (const [yearKey, groupsValue] of Object.entries(years)) { + const year = Number.parseInt(yearKey, 10); + const groups = object(groupsValue); + if (!Number.isInteger(year) || !groups) continue; + const definitions: TexasElectionDefinition[] = []; + for (const group of Object.values(groups)) { + const entries = object(group); + if (!entries) continue; + for (const candidate of Object.values(entries)) { + const item = object(candidate); + const id = numberValue(item?.ID); + const name = stringValue(item?.N); + if (id === undefined || !name) continue; + definitions.push({ + id, + name: name.replace(/\s+/g, " "), + year, + type: stringValue(item?.ET) ?? "unknown", + category: stringValue(item?.EC), + official: item?.O === "Y", + }); + } + } + electionsByYear[year] = definitions.sort((a, b) => a.id - b.id); + } + const cycleYear = Math.max(...Object.keys(electionsByYear).map(Number)); + if (!Number.isFinite(cycleYear)) { + throw new Error("Texas SOS discovery payload contains no election years"); + } + return { + cycleYear, + elections: electionsByYear[cycleYear] ?? [], + electionsByYear, + }; +} + +function cleanCandidateName(name: string): { + name: string; + incumbent: boolean; +} { + return { + name: name.replace(/\s*\(I\)\s*$/i, "").trim(), + incumbent: /\(I\)\s*$/i.test(name), + }; +} + +function parseChoice(value: unknown): TexasResultChoice | null { + const item = object(value); + const rawName = stringValue(item?.N); + if (!item || !rawName) return null; + const cleaned = cleanCandidateName(rawName); + return { + id: numberValue(item.ID ?? item.id), + name: cleaned.name, + party: stringValue(item.P), + incumbent: cleaned.incumbent, + votes: numberValue(item.V) ?? 0, + earlyVotes: numberValue(item.EV), + percent: numberValue(item.PE) ?? 0, + winner: false, + }; +} + +function propositionIdentity(title: string): { + number?: number; + resolution?: string; +} { + const number = /\bproposition\s+(\d+)\b/i.exec(title)?.[1]; + const resolution = /\(([SH]\.?\s*J\.?\s*R\.?)\s*(\d+)\)/i.exec(title); + return { + number: number ? Number.parseInt(number, 10) : undefined, + resolution: resolution + ? `${(resolution[1] ?? "").replace(/\s|\./g, "").toUpperCase()} ${resolution[2]}` + : undefined, + }; +} + +interface RaceSeed { + id: number; + title: string; + officeType?: string; + totalVotes: number; + choices: TexasResultChoice[]; +} + +function raceSeeds(raw: JsonObject): RaceSeed[] { + const groups: unknown[] = []; + const raceRoot = object(decodeJson(raw.Race)); + const officeTypes: unknown[] = Array.isArray(raceRoot?.OfficeTypes) + ? (raceRoot.OfficeTypes as unknown[]) + : []; + groups.push(...officeTypes); + for (const key of ["StateWide", "StateWideQ", "Districted", "Federal"]) { + const section = object(decodeJson(raw[key])); + if (section) groups.push(section); + } + + const byId = new Map(); + for (const groupValue of groups) { + const group = object(groupValue); + if (!group) continue; + const races = Array.isArray(group.Races) ? group.Races : []; + for (const raceValue of races) { + const race = object(raceValue); + if (!race) continue; + const id = numberValue(race.id ?? race.OID); + const title = stringValue(race.N); + if (id === undefined || !title || byId.has(id)) continue; + const choices = ( + Array.isArray(race.Candidates) + ? race.Candidates + : Object.values(object(race.C) ?? {}) + ) + .map(parseChoice) + .filter((choice): choice is TexasResultChoice => choice !== null) + .sort((a, b) => b.votes - a.votes); + const maxVotes = choices[0]?.votes ?? 0; + for (const choice of choices) + choice.winner = maxVotes > 0 && choice.votes === maxVotes; + byId.set(id, { + id, + title: title.replace(/\s+/g, " ").trim(), + officeType: stringValue(group.OfficeType ?? race.OT), + totalVotes: + numberValue(race.T) ?? + choices.reduce((sum, choice) => sum + choice.votes, 0), + choices, + }); + } + } + return [...byId.values()]; +} + +function reportingPercent( + reporting: TexasElectionReporting, +): number | undefined { + const pairs: [number | undefined, number | undefined][] = [ + [reporting.pollingPlacesReporting, reporting.pollingPlacesTotal], + [reporting.countiesReporting, reporting.countiesTotal], + [reporting.precinctsReporting, reporting.precinctsTotal], + ]; + for (const [done, total] of pairs) { + if (done !== undefined && total !== undefined && total > 0) { + return Math.min(100, Math.round((done / total) * 10_000) / 100); + } + } + return undefined; +} + +function isoDate(mmddyyyy: string | undefined): string | undefined { + if (!mmddyyyy || !/^\d{8}$/.test(mmddyyyy)) return undefined; + return `${mmddyyyy.slice(4)}-${mmddyyyy.slice(0, 2)}-${mmddyyyy.slice(2, 4)}`; +} + +function parseCountyResults( + raw: unknown, +): Map< + number, + { totals: TexasElectionTurnout; results: Map } +> { + const decoded = object(unwrapUpload(raw)); + const result = new Map< + number, + { totals: TexasElectionTurnout; results: Map } + >(); + if (!decoded) return result; + + for (const [countyIdKey, countyValue] of Object.entries(decoded)) { + const county = object(countyValue); + const countyId = Number.parseInt(countyIdKey, 10); + const name = stringValue(county?.N); + if (!county || !Number.isInteger(countyId) || !name) continue; + const summary = object(county.Summary); + const registeredVoters = numberValue(county.TV ?? summary?.RV); + const ballotsCast = numberValue(summary?.VC); + const turnoutPercent = + numberValue(summary?.VT) ?? + (registeredVoters && ballotsCast !== undefined + ? Math.round((ballotsCast / registeredVoters) * 10_000) / 100 + : undefined); + const byRace = new Map(); + const races = object(county.Races) ?? {}; + for (const raceValue of Object.values(races)) { + const race = object(raceValue); + const raceId = numberValue(race?.OID); + if (!race || raceId === undefined) continue; + const choices = Object.values(object(race.C) ?? {}) + .map(parseChoice) + .filter((choice): choice is TexasResultChoice => choice !== null) + .sort((a, b) => b.votes - a.votes); + const maxVotes = choices[0]?.votes ?? 0; + for (const choice of choices) + choice.winner = maxVotes > 0 && choice.votes === maxVotes; + const precinctsReporting = numberValue(race.PR); + const precinctsTotal = numberValue(race.TP); + byRace.set(raceId, [ + { + countyId, + county: name, + registeredVoters, + ballotsCast, + turnoutPercent, + reportingPercent: + precinctsReporting !== undefined && precinctsTotal + ? Math.min( + 100, + Math.round((precinctsReporting / precinctsTotal) * 10_000) / + 100, + ) + : undefined, + choices, + }, + ]); + } + result.set(countyId, { + totals: { registeredVoters, ballotsCast, turnoutPercent }, + results: byRace, + }); + } + return result; +} + +/** Normalize one SOS election payload, optionally including county totals. */ +export function parseTexasSosElection( + raw: unknown, + definition: TexasElectionDefinition, + countyRaw?: unknown, + today = new Date(), +): TexasSosElection { + const root = object(raw); + if (!root) + throw new Error(`Texas SOS election ${definition.id} is not an object`); + const home = object(decodeJson(root.Home)) ?? {}; + const reporting: TexasElectionReporting = { + countiesReporting: numberValue(object(home.CountiesReporting)?.CR), + countiesTotal: numberValue(object(home.CountiesReporting)?.CT), + precinctsReporting: numberValue(object(home.PrecinctsReporting)?.PR), + precinctsTotal: numberValue(object(home.PrecinctsReporting)?.PT), + pollingPlacesReporting: numberValue(object(home.PollingReporting)?.PLR), + pollingPlacesTotal: numberValue(object(home.PollingReporting)?.PLT), + }; + reporting.percentReporting = reportingPercent(reporting); + const date = isoDate(stringValue(home.ElecDate)); + const counties = parseCountyResults(countyRaw); + const countyRows = [...counties.values()]; + const registeredVoters = countyRows.reduce( + (sum, row) => sum + (row.totals.registeredVoters ?? 0), + 0, + ); + const ballotsCast = countyRows.reduce( + (sum, row) => sum + (row.totals.ballotsCast ?? 0), + 0, + ); + const citation: SourceCitation = { + sourceName: "Texas Secretary of State", + sourceUrl: TEXAS_RESULTS_URL, + official: true, + provider: TEXAS_SOS_PROVIDER, + }; + const contests = raceSeeds(root).map((race): TexasElectionContest => { + const identity = propositionIdentity(race.title); + const type = identity.number === undefined ? "candidate" : "referendum"; + const contestCounties: TexasCountyResult[] = []; + for (const county of counties.values()) { + contestCounties.push(...(county.results.get(race.id) ?? [])); + } + const forChoice = race.choices.find((choice) => /^for$/i.test(choice.name)); + const againstChoice = race.choices.find((choice) => + /^against$/i.test(choice.name), + ); + return { + id: `tx-sos:${definition.id}:${race.id}`, + sourceContestId: race.id, + type, + title: race.title, + officeType: race.officeType, + propositionNumber: identity.number, + resolution: identity.resolution, + totalVotes: race.totalVotes, + choices: race.choices, + counties: contestCounties, + outcome: + forChoice && againstChoice && race.totalVotes > 0 + ? forChoice.votes > againstChoice.votes + ? "adopted" + : "rejected" + : undefined, + citation, + }; + }); + const totalVotes = contests.reduce( + (sum, contest) => sum + contest.totalVotes, + 0, + ); + const electionDate = date ? new Date(`${date}T23:59:59Z`) : undefined; + const complete = (reporting.percentReporting ?? 0) >= 100; + const status = definition.official + ? "official" + : electionDate && electionDate > today && totalVotes === 0 + ? "upcoming" + : complete + ? "complete" + : totalVotes > 0 + ? "reporting" + : "upcoming"; + return { + id: definition.id, + name: definition.name, + date, + year: definition.year, + electionType: definition.type, + status, + official: definition.official, + lastUpdated: stringValue(home.LastUpdatedTime), + sourceVersion: stringValue(root.Version) ?? `election/${definition.id}`, + reporting, + turnout: + registeredVoters > 0 + ? { + registeredVoters, + ballotsCast, + turnoutPercent: + Math.round((ballotsCast / registeredVoters) * 10_000) / 100, + } + : undefined, + contests, + citation, + }; +} + +function compactText(value: string): string { + return value + .replace(/\u00ad/g, "") + .replace(/[ \t]+/g, " ") + .replace(/\s*\n\s*/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function section( + text: string, + start: RegExp, + ends: RegExp[], +): string | undefined { + const match = start.exec(text); + if (!match) return undefined; + const tail = text.slice(match.index + match[0].length); + let end = tail.length; + for (const pattern of ends) { + const next = pattern.exec(tail); + if (next && next.index < end) end = next.index; + } + const value = compactText(tail.slice(0, end)); + return value || undefined; +} + +function argumentList(text: string | undefined): string[] { + if (!text) return []; + const pieces = text.includes("β€’") ? text.split("β€’") : text.split(/\n\s*[-–]/); + return pieces.map(compactText).filter((item) => item.length >= 15); +} + +function fiscalSentences(...texts: (string | undefined)[]): string[] { + const seen = new Set(); + const output: string[] = []; + for (const text of texts) { + for (const sentence of (text ?? "").split(/(?<=[.!?])\s+/)) { + const value = compactText(sentence); + if ( + value.length >= 30 && + /\b(?:fiscal|tax|revenue|fund(?:ing)?|appropriat|cost|dollars?|\$)\b/i.test( + value, + ) && + !seen.has(value.toLowerCase()) + ) { + seen.add(value.toLowerCase()); + output.push(value); + } + if (output.length >= 8) return output; + } + } + return output; +} + +function tlcCitation(sourceUrl: string, page: number): SourceCitation { + return { + sourceName: "Texas Legislative Council", + sourceUrl: `${sourceUrl}#page=${page}`, + official: true, + provider: TEXAS_TLC_PROVIDER, + page, + }; +} + +/** Parse text-layer pages from either the 2023 or 2025 TLC report layout. */ +export function parseTexasTlcAnalysis( + pages: TlcTextPage[], + sourceUrl: string, +): TexasTlcSnapshotData { + const cover = compactText( + pages + .slice(0, 8) + .map((page) => page.text) + .join(" "), + ); + const year = Number.parseInt( + /(?:19|20)\d{2}/.exec(cover)?.[0] ?? String(new Date().getFullYear()), + 10, + ); + const electionDate = + /\b(November|May|September)\s+\d{1,2},\s+(?:19|20)\d{2}\b/i.exec( + cover, + )?.[0]; + const starts: { index: number; propositionNumber: number }[] = []; + for (let i = 0; i < pages.length; i++) { + const page = pages[i]; + if (!page) continue; + const text = page.text; + // The ballot language between the heading and SUMMARY ANALYSIS ranges from + // one short line to several wrapped lines (notably Props. 7, 12, 14, 17 in + // the 2025 layout). Keep the bound finite so table-of-contents pages cannot + // accidentally bridge into an unrelated heading. + const match = + /\bProposition\s+(\d+)\b[\s\S]{0,1200}\bSUMMARY\s+ANALYSIS\b/i.exec(text); + if (match?.[1]) + starts.push({ index: i, propositionNumber: Number(match[1]) }); + } + + const measures = starts.map((start, position): TexasTlcMeasure => { + const endIndex = (starts[position + 1]?.index ?? pages.length) - 1; + const text = pages + .slice(start.index, endIndex + 1) + .map((page) => page.text) + .join("\n"); + const heading = + /\bProposition\s+(\d+)\s*\(([SH]\.?\s*J\.?\s*R\.?)\s*(\d+)\)\s*/i.exec( + text, + ); + const afterHeading = heading + ? text.slice(heading.index + heading[0].length) + : text; + const ballotLanguage = compactText( + afterHeading.split(/\bSUMMARY\s+ANALYSIS\b/i)[0] ?? "", + ); + const summaryAnalysis = section(text, /\bSUMMARY\s+ANALYSIS\b/i, [ + /\bBACKGROUND(?:\s+AND\s+DETAILED\s+ANALYSIS)?\b/i, + /\bSUMMARY\s+OF\s+COMMENTS\b/i, + ]); + const background = section( + text, + /\bBACKGROUND(?:\s+AND\s+DETAILED\s+ANALYSIS)?\b/i, + [/\bSUMMARY\s+OF\s+COMMENTS\b/i, /\bCOMMENTS\s+BY\s+SUPPORTERS\b/i], + ); + const supporters = section(text, /\bComments\s+by\s+Supporters\s*:?/i, [ + /\bComments\s+by\s+Opponents\s*:?/i, + /\bText\s+of\s+[SH]\.?\s*J\.?\s*R\.?/i, + ]); + const opponents = section(text, /\bComments\s+by\s+Opponents\s*:?/i, [ + /\bText\s+of\s+[SH]\.?\s*J\.?\s*R\.?/i, + ]); + const startPage = pages[start.index]; + const endPage = pages[Math.max(start.index, endIndex)]; + if (!startPage || !endPage) { + throw new Error("TLC proposition page bounds are invalid"); + } + const pageStart = startPage.page; + const pageEnd = endPage.page; + const citation = tlcCitation(sourceUrl, pageStart); + const supporterArguments = argumentList(supporters); + const opponentArguments = argumentList(opponents); + const fiscalImplications = fiscalSentences(summaryAnalysis, background); + const diagnostics: string[] = []; + if (!ballotLanguage) diagnostics.push("missing ballot language"); + if (!summaryAnalysis) diagnostics.push("missing summary analysis"); + if (!background) diagnostics.push("missing background analysis"); + if (!supporterArguments.length) + diagnostics.push("missing supporter comments"); + if (!opponentArguments.length) + diagnostics.push("missing opponent comments"); + return { + propositionNumber: start.propositionNumber, + resolution: heading + ? `${(heading[2] ?? "").replace(/\s|\./g, "").toUpperCase()} ${heading[3]}` + : undefined, + ballotLanguage: ballotLanguage || undefined, + summaryAnalysis, + background, + supporterArguments, + opponentArguments, + fiscalImplications, + pageStart, + pageEnd, + citations: { + ballotLanguage: ballotLanguage ? citation : undefined, + summaryAnalysis: summaryAnalysis ? citation : undefined, + background: background ? citation : undefined, + supporterArguments: supporterArguments.length ? citation : undefined, + opponentArguments: opponentArguments.length ? citation : undefined, + fiscalImplications: fiscalImplications.length ? citation : undefined, + }, + diagnostics, + }; + }); + + return { + cycleYear: year, + electionDate, + publicationTitle: `Analyses of Proposed Constitutional Amendments (${year})`, + sourceUrl, + measures, + }; +} + +function asSosData( + value: Record, +): TexasSosSnapshotData | null { + return typeof value.cycleYear === "number" && Array.isArray(value.elections) + ? (value as unknown as TexasSosSnapshotData) + : null; +} + +function asTlcData( + value: Record, +): TexasTlcSnapshotData | null { + return typeof value.cycleYear === "number" && Array.isArray(value.measures) + ? (value as unknown as TexasTlcSnapshotData) + : null; +} + +/** Read and join only the current SOS cycle plus the latest TLC amendment cycle. */ +export async function getTexasCurrentElectionData(): Promise { + const rows = await db + .select() + .from(ElectionSourceSnapshot) + .where( + and( + eq(ElectionSourceSnapshot.jurisdiction, "TX"), + eq(ElectionSourceSnapshot.scope, TEXAS_CURRENT_SCOPE), + ), + ) + .orderBy(desc(ElectionSourceSnapshot.cycleYear)); + const sosRows = rows.filter((row) => row.provider === TEXAS_SOS_PROVIDER); + const tlcRow = rows.find((row) => row.provider === TEXAS_TLC_PROVIDER); + const tlc = tlcRow ? asTlcData(tlcRow.data) : null; + const currentSos = sosRows.map((row) => asSosData(row.data)).find(Boolean); + if (!currentSos && !tlc) return null; + const amendmentSos = tlc + ? sosRows + .map((row) => asSosData(row.data)) + .find((data) => data?.cycleYear === tlc.cycleYear) + : null; + const amendmentElection = amendmentSos?.elections.find((election) => + election.contests.some((contest) => contest.type === "referendum"), + ); + const byProposition = new Map( + amendmentElection?.contests.flatMap((contest) => + contest.propositionNumber === undefined + ? [] + : ([[contest.propositionNumber, contest]] as const), + ) ?? [], + ); + const fetchedAt = rows.reduce( + (latest, row) => (row.fetchedAt > latest ? row.fetchedAt : latest), + new Date(0), + ); + return { + jurisdiction: "TX", + cycleYear: currentSos?.cycleYear ?? tlc?.cycleYear ?? 0, + elections: currentSos?.elections ?? [], + constitutionalAmendments: tlc + ? { + cycleYear: tlc.cycleYear, + electionDate: tlc.electionDate, + publicationTitle: tlc.publicationTitle, + measures: tlc.measures.map((measure) => { + const contest = byProposition.get(measure.propositionNumber); + return { + ...measure, + result: + contest && amendmentElection + ? { + outcome: contest.outcome, + totalVotes: contest.totalVotes, + choices: contest.choices, + citation: contest.citation, + status: amendmentElection.status, + asOf: amendmentElection.lastUpdated, + } + : undefined, + }; + }), + citation: tlcCitation(tlc.sourceUrl, 1), + } + : undefined, + diagnostics: rows.flatMap((row) => row.diagnostics), + fetchedAt: fetchedAt.toISOString(), + }; +} diff --git a/packages/api/src/lib/votesmart.ts b/packages/api/src/lib/votesmart.ts index 3e40ac88..0528ea80 100644 --- a/packages/api/src/lib/votesmart.ts +++ b/packages/api/src/lib/votesmart.ts @@ -221,6 +221,7 @@ export async function enrichFromVoteSmart( referendumTitle: string, stateAbbrev: string, electionYear: number, + electionDate?: string, ): Promise { if (!getApiKey()) return null; @@ -238,6 +239,18 @@ export async function enrichFromVoteSmart( let best: VoteSmartMeasure | null = null; let bestScore = 0; for (const m of measures) { + if (electionDate && m.electionDate) { + const requested = new Date(electionDate); + const candidate = new Date(m.electionDate); + if ( + !Number.isNaN(requested.getTime()) && + !Number.isNaN(candidate.getTime()) && + requested.toISOString().slice(0, 10) !== + candidate.toISOString().slice(0, 10) + ) { + continue; + } + } const score = titleSimilarity(referendumTitle, m.title); if (score > bestScore) { bestScore = score; diff --git a/packages/api/src/router/civic.ts b/packages/api/src/router/civic.ts index a921b1c6..dd04f3fd 100644 --- a/packages/api/src/router/civic.ts +++ b/packages/api/src/router/civic.ts @@ -10,6 +10,7 @@ import { getVoterInfo, } from "../lib/civic"; import { getElectedOfficials } from "../lib/elected-officials"; +import { getTexasCurrentElectionData } from "../lib/texas-election-data"; import { publicProcedure } from "../trpc"; const STATEWIDE_OFFICE = z.enum( @@ -42,6 +43,25 @@ export const civicRouter = { } }), + /** + * Current-cycle Texas statewide election data from the persisted SOS/TLC + * handoff. This intentionally exposes no historical browsing parameter. + */ + getTexasCurrentElection: publicProcedure.query(async () => { + try { + return await getTexasCurrentElectionData(); + } catch (error) { + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: + error instanceof Error + ? error.message + : "Failed to read current Texas election data", + cause: error, + }); + } + }), + /** * Get live California statewide election results (Secretary of State feed). * Defaults to the marquee races (governor + secretary of state) when no diff --git a/packages/db/migrations/add_election_source_snapshots.sql b/packages/db/migrations/add_election_source_snapshots.sql new file mode 100644 index 00000000..9d132a99 --- /dev/null +++ b/packages/db/migrations/add_election_source_snapshots.sql @@ -0,0 +1,23 @@ +-- Provider-neutral handoff for current-cycle official election sources. +-- Texas SOS facts and TLC explanatory text are stored in separate provider +-- rows so their citations cannot be accidentally collapsed. +CREATE TABLE IF NOT EXISTS election_source_snapshot ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + jurisdiction varchar(20) NOT NULL, + cycle_year integer NOT NULL, + provider varchar(50) NOT NULL, + scope varchar(50) DEFAULT 'current' NOT NULL, + source_version varchar(150) NOT NULL, + content_hash varchar(64) NOT NULL, + data jsonb NOT NULL, + diagnostics jsonb DEFAULT '[]'::jsonb NOT NULL, + source_urls jsonb DEFAULT '[]'::jsonb NOT NULL, + fetched_at timestamp with time zone NOT NULL, + created_at timestamp DEFAULT now() NOT NULL, + updated_at timestamp with time zone, + CONSTRAINT election_source_snapshot_unique + UNIQUE (jurisdiction, cycle_year, provider, scope) +); + +CREATE INDEX IF NOT EXISTS election_snapshot_current_cycle_idx + ON election_source_snapshot (jurisdiction, scope, cycle_year); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 3c990bae..b3786c6a 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -308,6 +308,44 @@ export const ElectionRecord = pgTable( }), ); +// Provider-neutral snapshots for official election feeds that do not map +// cleanly onto the legacy relational election tables. Each provider owns one +// idempotent row per jurisdiction/cycle/scope; readers normalize and join rows +// without mixing their source attribution. +export const ElectionSourceSnapshot = pgTable( + "election_source_snapshot", + (t) => ({ + id: t.uuid().notNull().primaryKey().defaultRandom(), + jurisdiction: t.varchar({ length: 20 }).notNull(), + cycleYear: t.integer().notNull(), + provider: t.varchar({ length: 50 }).notNull(), + scope: t.varchar({ length: 50 }).notNull().default("current"), + sourceVersion: t.varchar({ length: 150 }).notNull(), + contentHash: t.varchar({ length: 64 }).notNull(), + data: t.jsonb().$type>().notNull(), + diagnostics: t.jsonb().$type().notNull().default([]), + sourceUrls: t.jsonb().$type().notNull().default([]), + fetchedAt: t.timestamp({ mode: "date", withTimezone: true }).notNull(), + createdAt: t.timestamp().defaultNow().notNull(), + updatedAt: t + .timestamp({ mode: "date", withTimezone: true }) + .$onUpdateFn(() => sql`now()`), + }), + (table) => ({ + uniqueProviderSnapshot: unique().on( + table.jurisdiction, + table.cycleYear, + table.provider, + table.scope, + ), + currentCycleIdx: index("election_snapshot_current_cycle_idx").on( + table.jurisdiction, + table.scope, + table.cycleYear, + ), + }), +); + // Role descriptions β€” reusable across elections, keyed by (role, level) export const RoleDescriptionRecord = pgTable( "role_description", diff --git a/packages/env/src/registry.ts b/packages/env/src/registry.ts index 6686f519..9eda1aa5 100644 --- a/packages/env/src/registry.ts +++ b/packages/env/src/registry.ts @@ -85,6 +85,7 @@ const scraperSourceLimitDefinitions = [ ["SCOTUS_MAX_ITEMS", "CourtListener opinion clusters per run.", "50"], ["SCC_CVIG_MAX_ITEMS", "Santa Clara voter-guide PDFs per run.", "10"], ["CA_SOS_MAX_ITEMS", "California SOS office pages per run.", "9"], + ["TX_SOS_MAX_ITEMS", "Texas SOS election payloads per run.", "12"], ] as const; export const envRegistry = [