Add North Carolina and Texas civic data scrapers

apps/expo/src/components/UpcomingMeetingsSection.tsx

@@ -7,16 +7,16 @@ import {
 import { FontAwesome } from "@expo/vector-icons";
 import { useQuery } from "@tanstack/react-query";
 
-import type { LegistarMeeting } from "@acme/api/integrations/legistar";
+import type { RouterOutputs } from "@acme/api";
 
 import { Text, View } from "~/components/Themed";
 import { fontBody, fontEditorial, fontSize, rd, sp, useTheme } from "~/styles";
 import { trpc } from "~/utils/api";
 
+type Meeting = RouterOutputs["localGovernment"]["listMeetings"][number];
+
 interface UpcomingMeetingsSectionProps {
-  onMeetingPress?: (
-    meeting: LegistarMeeting & { jurisdiction: string },
-  ) => void;
+  onMeetingPress?: (meeting: Meeting) => void;
 }
 
 function formatDate(iso: string): string {
@@ -34,7 +34,7 @@ export function UpcomingMeetingsSection({
   const { theme } = useTheme();
 
   const meetingsQuery = useQuery(
-    trpc.legistar.getMeetings.queryOptions({ daysAhead: 30 }),
+    trpc.localGovernment.listMeetings.queryOptions({ daysAhead: 90 }),
   );
 
   return (
@@ -47,30 +47,43 @@ export function UpcomingMeetingsSection({
 
       {meetingsQuery.data?.slice(0, 8).map((meeting, index) => (
         <TouchableOpacity
-          key={`${meeting.EventId}-${index}`}
+          key={`${meeting.source}-${meeting.externalId}-${index}`}
           style={[styles.card, { backgroundColor: theme.card }]}
-          onPress={() => onMeetingPress?.(meeting)}
+          onPress={() =>
+            onMeetingPress
+              ? onMeetingPress(meeting)
+              : void Linking.openURL(meeting.canonicalUrl)
+          }
           activeOpacity={0.8}
         >
           <View style={styles.cardAccent} />
           <View style={styles.cardContent}>
             <View style={styles.meta}>
               <Text style={styles.jurisdiction}>{meeting.jurisdiction}</Text>
-              <Text style={styles.date}>{formatDate(meeting.EventDate)}</Text>
+              <Text style={styles.date}>
+                {formatDate(meeting.startsAt.toString())}
+              </Text>
             </View>
             <Text style={styles.title} numberOfLines={2}>
-              {meeting.EventBodyName}
+              {meeting.isCancelled ? "Cancelled: " : ""}
+              {meeting.title}
             </Text>
-            {meeting.EventLocation && (
+            {meeting.location && (
               <Text style={styles.location} numberOfLines={1}>
-                {meeting.EventLocation}
+                {meeting.location}
               </Text>
             )}
             <View style={styles.icons}>
-              {meeting.EventAgendaFile && (
+              {meeting.documents.find(
+                (document) => document.type === "agenda",
+              ) && (
                 <TouchableOpacity
                   onPress={() =>
-                    void Linking.openURL(meeting.EventAgendaFile ?? "")
+                    void Linking.openURL(
+                      meeting.documents.find(
+                        (document) => document.type === "agenda",
+                      )?.url ?? "",
+                    )
                   }
                   hitSlop={8}
                 >
@@ -81,11 +94,9 @@ export function UpcomingMeetingsSection({
                   />
                 </TouchableOpacity>
               )}
-              {meeting.EventVideoPath && (
+              {meeting.videoUrl && (
                 <TouchableOpacity
-                  onPress={() =>
-                    void Linking.openURL(meeting.EventVideoPath ?? "")
-                  }
+                  onPress={() => void Linking.openURL(meeting.videoUrl ?? "")}
                   hitSlop={8}
                 >
                   <FontAwesome
@@ -95,10 +106,16 @@ export function UpcomingMeetingsSection({
                   />
                 </TouchableOpacity>
               )}
-              {meeting.EventMinutesFile && (
+              {meeting.documents.find(
+                (document) => document.type === "minutes",
+              ) && (
                 <TouchableOpacity
                   onPress={() =>
-                    void Linking.openURL(meeting.EventMinutesFile ?? "")
+                    void Linking.openURL(
+                      meeting.documents.find(
+                        (document) => document.type === "minutes",
+                      )?.url ?? "",
+                    )
                   }
                   hitSlop={8}
                 >

apps/scraper/README.md

@@ -4,15 +4,20 @@ Pulls in government content like bills, court cases, and White House content and
 
 ## Active data sources
 
-Only these five are registered and run by `all`:
-
-| CLI name            | Source and data fetched                                                                    | Stored/used as                                                                                        |
-| ------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- |
-| `federalregister`   | Federal Register API presidential documents, then each document's body HTML                | `government_content`; AI article/summary and feed-image enrichment                                    |
-| `congress`          | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment                                            |
-| `scotus`            | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court        | `court_case`; powers court content and AI/feed enrichment                                             |
-| `scc-cvig`          | Hand-configured Santa Clara County voter-guide PDFs                                        | Candidate statements in `CivicApiCache`; the API matches statements to candidates                     |
-| `ca-sos-statements` | California SOS statewide-office candidate-statement pages                                  | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source |
+These data sources are registered and run by `all`:
+
+| CLI name                 | Source and data fetched                                                                    | Stored/used as                                                                                        |
+| ------------------------ | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- |
+| `federalregister`        | Federal Register API presidential documents, then each document's body HTML                | `government_content`; AI article/summary and feed-image enrichment                                    |
+| `durham-bocc`            | Durham County's official Legistar API (current election cycle only)                        | Provider-neutral meetings, agenda items, actions, votes, and official document links; no AI required  |
+| `congress`               | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment                                            |
+| `scotus`                 | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court        | `court_case`; powers court content and AI/feed enrichment                                             |
+| `scc-cvig`               | Hand-configured Santa Clara County voter-guide PDFs                                        | Candidate statements in `CivicApiCache`; the API matches statements to candidates                     |
+| `ca-sos-statements`      | California SOS statewide-office candidate-statement pages                                  | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source |
+| `ncsbe`                  | Current-cycle NCSBE candidate CSV, referendum PDFs, and result ZIPs                        | Provider-neutral election tables; powers `civic.getNcElectionData` with exact file provenance         |
+| `texas-legislature`      | Texas Legislative Council anonymous FTP: current-session history XML and bulk documents    | State-aware `bill` rows; read through `content.texasBills` and `content.getById`                      |
+| `texas-current-election` | Texas SOS structured election feed and TLC amendment analyses                              | Current-cycle snapshots; powers `civic.getTexasCurrentElection` and measure enrichment                |
+| `cedar-park-council`     | Cedar Park's CivicEngage City Council page and its official Municode Meetings embed        | Provider-neutral local meetings, documents, agenda items, motions, outcomes, and votes                |
 
 `vote411`, `ca-lao-fiscal`, and `ca-vig-archive` remain under
 `src/scrapers/disabled/` and do not run. Their caches had no application
@@ -57,6 +62,7 @@ work:
 | `COURTLISTENER_API_KEY`                      | Optional                                | Higher CourtListener limits for `scotus`.                                                                          |
 | `GOOGLE_API_KEY` / `GOOGLE_SEARCH_ENGINE_ID` | Optional pair                           | Google Custom Search article thumbnails.                                                                           |
 | `GOOGLE_GENERATIVE_AI_API_KEY`               | Optional                                | Gemini vision fallback for `scc-cvig` PDF extraction.                                                              |
+| `OPEN_STATES_API_KEY`                        | Optional                                | Adds an exact Open States bill ID when Texas jurisdiction/session/identifier match.                                |
 
 See [the launch environment guide](../../docs/launch.md) for the complete
 per-scraper matrix, provider setup links, defaults, and production guidance.
@@ -128,6 +134,11 @@ CONGRESS_MAX_ITEMS=10 pnpm --filter @acme/scraper run start congress
 | `SCOTUS_MAX_ITEMS`              |      50 | CourtListener opinion clusters                      |
 | `SCC_CVIG_MAX_ITEMS`            |      10 | Voter-guide PDF documents                           |
 | `CA_SOS_MAX_ITEMS`              |       9 | Statewide-office candidate-statement pages          |
+| `NCSBE_MAX_ITEMS`               |       4 | Current-cycle candidate/referendum/result files     |
+| `TEXAS_LEGISLATURE_MAX_ITEMS`   |     100 | Bills from the latest Texas bulk session            |
+| `TX_SOS_MAX_ITEMS`              |      12 | Current-cycle Texas SOS election payloads           |
+| `CEDAR_PARK_COUNCIL_MAX_ITEMS`  |     100 | Council meetings (after the 12-month cutoff)        |
+| `DURHAM_BOCC_MAX_ITEMS`         |     100 | Current-cycle Durham County BOCC meetings           |
 | `SCRAPER_MAX_NEW_ITEMS_PER_RUN` |      10 | New records receiving expensive AI/image enrichment |
 
 These are per-run limits, not durable calendar-day quotas. Schedule one run per
@@ -137,6 +148,33 @@ invocation gets a fresh allowance. Source limits cap API/page work;
 bills that require a generated description are deferred before insertion;
 other content may still be stored raw for later backfill.
 
+The NCSBE integration is intentionally current-cycle only and excludes voter
+history plus candidate contact/address fields. See
+[`docs/ncsbe-election-data.md`](../../docs/ncsbe-election-data.md) for discovery,
+idempotency, provenance, API, and deterministic Civic-matching details.
+
+## Cedar Park City Council (`civicengage.ts`)
+
+Cedar Park's public site is CivicEngage, but the City Council records page now
+embeds a keyless Municode Meetings publish page. The adapter follows that
+official embed, keeps a 12-month Council-only window, downloads at most two
+documents concurrently, and deterministically parses PDF text. AI/vision is not
+used. Agendas, packets, minutes, and later document URLs are versioned beneath
+one provider-neutral meeting record; unchanged reruns produce the same natural
+keys and checksums.
+
+```bash
+pnpm --filter @acme/scraper run start cedar-park-council --max-items 1
+```
+
+To add a second CivicEngage jurisdiction using the same kind of official
+Municode embed, add a `CivicEngageJurisdictionConfig` beside
+`cedarParkCouncilSource` with its CivicEngage host/path, IANA timezone,
+Municode `cid`/`ppid`, and governing-body matcher, then instantiate the same
+discovery/parser pipeline. If its records page uses Agenda Center, Legistar, or
+another provider, implement that provider behind the same local-government
+persistence contract instead.
+
 ---
 
 ## Congress bills (`congress.ts`)
@@ -157,6 +195,34 @@ await scrapeCongress({
 
 ---
 
+## Texas Legislature Online (`texas-legislature.ts`)
+
+Uses only the Texas Legislative Council's anonymous FTP service at
+`ftp.legis.state.tx.us`; it does not crawl interactive TLO bill pages. The job:
+
+- discovers the newest session directory under `/bills` and rejects a stale
+  `TEXAS_LEGISLATURE_SESSION` assertion (official codes look like `89R` or
+  `892`);
+- parses bill-history XML for identity, caption, sponsors, subjects, actions,
+  structured votes when present, and document metadata;
+- downloads bill text, analyses, and fiscal notes from the matching FTP bulk
+  HTML paths and stores their extracted text alongside official HTML/PDF links;
+- optionally stores an exact Open States ID without using Open States as the
+  legislative data source.
+
+Run a small current-session import:
+
+```bash
+pnpm --filter @acme/scraper run start texas-legislature --max-items 10
+```
+
+Apply `packages/db/migrations/add_state_legislation_fields.sql` before the first
+run. The public `content.texasBills` procedure lists only the newest persisted
+Texas session; `content.getById` returns its documents, actions, and votes. This
+work deliberately does not provide historical-session browsing or backfills.
+
+---
+
 ## Court cases (`scotus.ts`)
 
 Uses the [CourtListener API](https://www.courtlistener.com/api/) — free, works without a key. Fetches recent opinions and pulls in the plain-text opinion content for AI article generation.
@@ -184,6 +250,9 @@ All scrapers call into `src/utils/db/operations.ts`. Each time a bill or case is
 - If the **content changed** → regenerates the article
 - If **nothing changed** → backfills any missing AI summary/article/thumbnail fields, otherwise skips AI generation
 
+The Texas importer passes `skipEnrichment` so official source data is persisted
+without AI summaries, generated imagery, or videos.
+
 Set `SCRAPER_FORCE_AI_REGEN=1` to force a full AI refresh even when the record already has AI content.
 
 For a new bill whose description must be generated, the summary is now created

apps/scraper/package.json

@@ -13,6 +13,7 @@
     "@ai-sdk/openai-compatible": "2.0.59",
     "@openrouter/ai-sdk-provider": "^2.10.0",
     "ai": "^6.0.141",
+    "basic-ftp": "^6.0.1",
     "cheerio": "^1.2.0",
     "consola": "^3.4.2",
     "drizzle-orm": "^0.45.2",

apps/scraper/src/fixtures/ncsbe/candidates-2026.csv

@@ -0,0 +1,3 @@
+"election_dt","county_name","contest_name","name_on_ballot","party_candidate","has_primary","is_partisan","vote_for","term"
+"03/03/2026","DURHAM","US HOUSE OF REPRESENTATIVES DISTRICT 04","Nida Allam","DEM","TRUE","TRUE","1","2"
+"03/03/2026","WAKE","WAKE COUNTY BOARD OF COMMISSIONERS AT-LARGE","Marguerite Creel","DEM","TRUE","TRUE","2","4"

apps/scraper/src/fixtures/ncsbe/referendums-2026.txt

@@ -0,0 +1,12 @@
+REFERENDUM CHOICES LIST GROUPED BY REFERENDUM
+GATES BOARD OF ELECTIONS
+CRITERIA: Election: 03/03/2026, County: ALL COUNTIES
+GATES
+GATES COUNTY LOCAL SALES AND USE TAX REFERENDUM
+For Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes.
+Against Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes.
+GRANVILLE BOARD OF ELECTIONS
+GRANVILLE
+GRANVILLE COUNTY LOCAL SALES AND USE TAX REFERENDUM
+For Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes.
+Against Local sales and use tax at the rate of one-quarter percent (0.25%) in addition to all other State and local sales and use taxes.

apps/scraper/src/fixtures/ncsbe/results-2024.tsv

@@ -0,0 +1,3 @@
+county	election_date	precinct	contest_group_id	contest_type	contest_name	candidate	party	vote_for	election_day	early_voting	absentee_by_mail	provisional	total_votes	real_precinct
+DURHAM	11/05/2024	01	1373	S	US PRESIDENT	Kamala D. Harris	DEM	1	100	200	30	2	332	Y
+WAKE	11/05/2024	01-01	1373	S	US PRESIDENT	Kamala D. Harris	DEM	1	150	250	40	3	443	Y

apps/scraper/src/fixtures/ncsbe/results-2026.tsv

@@ -0,0 +1,3 @@
+County	Election Date	Precinct	Contest Group ID	Contest Type	Contest Name	Choice	Choice Party	Vote For	Election Day	Early Voting	Absentee by Mail	Provisional	Total Votes	Real Precinct
+DURHAM	03/03/2026	04	2114	S	US HOUSE OF REPRESENTATIVES DISTRICT 04 (DEM)	Nida Allam	DEM	1	371	0	0	0	371	Y
+WAKE	03/03/2026	06-05	1	C	WAKE COUNTY BOARD OF COMMISSIONERS AT-LARGE (DEM)	Marguerite Creel	DEM	2	12	0	0	0	12	Y

apps/scraper/src/scraper-contracts.ts

@@ -1,15 +1,27 @@
 import type { ScraperEnvContract } from "@acme/env";
 
 import { caSosStatementsConfig } from "./scrapers/ca-sos-statements.config.js";
+import { cedarParkCouncilConfig } from "./scrapers/civicengage.config.js";
 import { congressConfig } from "./scrapers/congress.config.js";
+import { durhamBoccConfig } from "./scrapers/durham-bocc.config.js";
+import { durhamOnBaseConfig } from "./scrapers/durham-onbase.config.js";
 import { federalregisterConfig } from "./scrapers/federalregister.config.js";
+import { ncsbeConfig } from "./scrapers/ncsbe.config.js";
 import { sccCvigConfig } from "./scrapers/scc-cvig.config.js";
 import { scotusConfig } from "./scrapers/scotus.config.js";
+import { texasCurrentElectionConfig } from "./scrapers/texas-current-election.config.js";
+import { texasLegislatureConfig } from "./scrapers/texas-legislature.config.js";
 
 export const scraperContracts: readonly ScraperEnvContract[] = [
   federalregisterConfig,
+  durhamBoccConfig,
   congressConfig,
   scotusConfig,
   sccCvigConfig,
   caSosStatementsConfig,
+  ncsbeConfig,
+  texasCurrentElectionConfig,
+  texasLegislatureConfig,
+  cedarParkCouncilConfig,
+  durhamOnBaseConfig,
 ];

apps/scraper/src/scrapers.ts

@@ -1,14 +1,26 @@
 import type { Scraper } from "./utils/types.js";
 import { caSosStatements } from "./scrapers/ca-sos-statements.js";
+import { cedarParkCouncil } from "./scrapers/civicengage.js";
 import { congress } from "./scrapers/congress.js";
+import { durhamBocc } from "./scrapers/durham-bocc.js";
+import { durhamOnBase } from "./scrapers/durham-onbase.js";
 import { federalregister } from "./scrapers/federalregister.js";
+import { ncsbe } from "./scrapers/ncsbe.js";
 import { sccCvig } from "./scrapers/scc-cvig.js";
 import { scotus } from "./scrapers/scotus.js";
+import { texasCurrentElection } from "./scrapers/texas-current-election.js";
+import { texasLegislature } from "./scrapers/texas-legislature.js";
 
 export const scrapers: readonly Scraper[] = [
   federalregister,
+  durhamBocc,
   congress,
   scotus,
   sccCvig,
   caSosStatements,
+  ncsbe,
+  texasCurrentElection,
+  texasLegislature,
+  cedarParkCouncil,
+  durhamOnBase,
 ];

apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html

@@ -0,0 +1,6 @@
+<html><body>
+<table><tr><td><a href="javascript:loadAgendaItem(21943,true);">Consent Agenda</a></td></tr></table>
+<table><tr><td><a href="javascript:loadAgendaItem(47768,false);">1. Approval of City Council Minutes</a></td></tr><tr><td>To approve the listed minutes. <b>[Approved by Vote: 7/0]</b></td></tr></table>
+<table><tr><td><a href="javascript:loadAgendaItem(21948,true);">General Business Agenda</a></td></tr></table>
+<table><tr><td><a href="javascript:loadAgendaItem(47875,false);">21. Consolidated Annexation – 4802 Cheek Road</a></td></tr><tr><td>Motion 1: To adopt the ordinance. <b>[FAILED by Vote: 0/7]</b> <b>Ayes: None.</b> <b>Nays: Mayor Williams, Council Member Baker.</b> Motion 2: Consider rezoning. <b>No vote taken.</b></td></tr></table>
+</body></html>