diff --git a/.env.example b/.env.example index ee61d50..c548c7c 100644 --- a/.env.example +++ b/.env.example @@ -75,6 +75,9 @@ EXPO_PUBLIC_API_URL=http://localhost:3000 # Get it: https://openstates.org/accounts/profile/ # OPEN_STATES_API_KEY= +# Optional per-run cap for changed Missouri General Assembly bills (default 100). +# MISSOURI_LEGISLATURE_MAX_ITEMS=100 + # Vote Smart key for candidate and state-measure enrichment. # Get it: https://votesmart.org/share/api # VOTE_SMART_API_KEY= diff --git a/apps/scraper/README.md b/apps/scraper/README.md index aca4ed6..cd18c96 100644 --- a/apps/scraper/README.md +++ b/apps/scraper/README.md @@ -16,6 +16,7 @@ These data sources are registered and run by `all`: | `ca-sos-statements` | California SOS statewide-office candidate-statement pages | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source | | `ncsbe` | Current-cycle NCSBE candidate CSV, referendum PDFs, and result ZIPs | Provider-neutral election tables; powers `civic.getNcElectionData` with exact file provenance | | `texas-legislature` | Texas Legislative Council anonymous FTP: current-session history XML and bulk documents | State-aware `bill` rows; read through `content.texasBills` and `content.getById` | +| `missouri-legislature` | Missouri House current-session XML exports | Shared state-aware `bill` rows; read through `content.stateBills` and `content.getById` | | `texas-current-election` | Texas SOS structured election feed and TLC amendment analyses | Current-cycle snapshots; powers `civic.getTexasCurrentElection` and measure enrichment | | `cedar-park-council` | Cedar Park's CivicEngage City Council page and its official Municode Meetings embed | Provider-neutral local meetings, documents, agenda items, motions, outcomes, and votes | @@ -62,7 +63,7 @@ work: | `COURTLISTENER_API_KEY` | Optional | Higher CourtListener limits for `scotus`. | | `GOOGLE_API_KEY` / `GOOGLE_SEARCH_ENGINE_ID` | Optional pair | Google Custom Search article thumbnails. | | `GOOGLE_GENERATIVE_AI_API_KEY` | Optional | Gemini vision fallback for `scc-cvig` PDF extraction. | -| `OPEN_STATES_API_KEY` | Optional | Adds an exact Open States bill ID when Texas jurisdiction/session/identifier match. | +| `OPEN_STATES_API_KEY` | Optional | Adds an exact Open States bill ID when Texas or Missouri jurisdiction/session/identifier match. | See [the launch environment guide](../../docs/launch.md) for the complete per-scraper matrix, provider setup links, defaults, and production guidance. @@ -127,19 +128,20 @@ pnpm --filter @acme/scraper run start congress --max-items 10 CONGRESS_MAX_ITEMS=10 pnpm --filter @acme/scraper run start congress ``` -| Variable | Default | Counts | -| ------------------------------- | ------: | --------------------------------------------------- | -| `FEDERALREGISTER_MAX_ITEMS` | 20 | Presidential documents | -| `CONGRESS_MAX_ITEMS` | 100 | Bills | -| `SCOTUS_MAX_ITEMS` | 50 | CourtListener opinion clusters | -| `SCC_CVIG_MAX_ITEMS` | 10 | Voter-guide PDF documents | -| `CA_SOS_MAX_ITEMS` | 9 | Statewide-office candidate-statement pages | -| `NCSBE_MAX_ITEMS` | 4 | Current-cycle candidate/referendum/result files | -| `TEXAS_LEGISLATURE_MAX_ITEMS` | 100 | Bills from the latest Texas bulk session | -| `TX_SOS_MAX_ITEMS` | 12 | Current-cycle Texas SOS election payloads | -| `CEDAR_PARK_COUNCIL_MAX_ITEMS` | 100 | Council meetings (after the 12-month cutoff) | -| `DURHAM_BOCC_MAX_ITEMS` | 100 | Current-cycle Durham County BOCC meetings | -| `SCRAPER_MAX_NEW_ITEMS_PER_RUN` | 10 | New records receiving expensive AI/image enrichment | +| Variable | Default | Counts | +| -------------------------------- | ------: | --------------------------------------------------- | +| `FEDERALREGISTER_MAX_ITEMS` | 20 | Presidential documents | +| `CONGRESS_MAX_ITEMS` | 100 | Bills | +| `SCOTUS_MAX_ITEMS` | 50 | CourtListener opinion clusters | +| `SCC_CVIG_MAX_ITEMS` | 10 | Voter-guide PDF documents | +| `CA_SOS_MAX_ITEMS` | 9 | Statewide-office candidate-statement pages | +| `NCSBE_MAX_ITEMS` | 4 | Current-cycle candidate/referendum/result files | +| `TEXAS_LEGISLATURE_MAX_ITEMS` | 100 | Bills from the latest Texas bulk session | +| `MISSOURI_LEGISLATURE_MAX_ITEMS` | 100 | Changed bills from active Missouri sessions | +| `TX_SOS_MAX_ITEMS` | 12 | Current-cycle Texas SOS election payloads | +| `CEDAR_PARK_COUNCIL_MAX_ITEMS` | 100 | Council meetings (after the 12-month cutoff) | +| `DURHAM_BOCC_MAX_ITEMS` | 100 | Current-cycle Durham County BOCC meetings | +| `SCRAPER_MAX_NEW_ITEMS_PER_RUN` | 10 | New records receiving expensive AI/image enrichment | These are per-run limits, not durable calendar-day quotas. Schedule one run per day to obtain a daily cap. If the scheduler retries or runs multiple times, each @@ -223,6 +225,38 @@ work deliberately does not provide historical-session browsing or backfills. --- +## Missouri General Assembly (`missouri-legislature.ts`) + +Discovers the active regular and any enabled special sessions from the official +[`SessionSet.js`](https://documents.house.mo.gov/SessionSet.js); no session code +is configured or hard-coded. Each run observes +the House's no-more-than-once-per-30-minutes rule through a durable database +guard, fetches `BillList.XML`, and downloads only individual House bill XML rows +whose `LastTimeRun` changed. At most two bill XML requests run concurrently. + +`SenateActList.XML` is also imported, but it is explicitly incomplete: it only +contains Senate bills with House activity. Their latest shared `bill.versions` +entry carries `changes = senate_with_house_actions_only`. Official actions, sponsors, +committees, proposed effective dates, roll-call totals, bill versions, +summaries, fiscal notes, amendments, veto letters, and witness documents are +stored in the shared `bill` contract. `OPEN_STATES_API_KEY` optionally adds an +exact identity match and is never required for official ingestion. + +```bash +pnpm --filter @acme/scraper run start missouri-legislature --max-items 10 +``` + +`content.stateBills({ stateCode: "MO" })` returns the active rows retained by +the latest official session descriptor; the scraper removes prior Missouri +sessions, so historical browsing is not provided. + +Source format and coverage references: [XML export overview](https://documents.house.mo.gov/), +[BillList fields](https://documents.house.mo.gov/XMLBillList.html), +[individual bill fields](https://documents.house.mo.gov/XMLBillExports.html), +and [Senate bills with House actions](https://documents.house.mo.gov/XMLSenateBillsWithHouseActions.html). + +--- + ## Court cases (`scotus.ts`) Uses the [CourtListener API](https://www.courtlistener.com/api/) — free, works without a key. Fetches recent opinions and pulls in the plain-text opinion content for AI article generation. diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index 5e78aa8..aceeaf7 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -6,6 +6,7 @@ import { congressConfig } from "./scrapers/congress.config.js"; import { durhamBoccConfig } from "./scrapers/durham-bocc.config.js"; import { durhamOnBaseConfig } from "./scrapers/durham-onbase.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; +import { missouriLegislatureConfig } from "./scrapers/missouri-legislature.config.js"; import { ncsbeConfig } from "./scrapers/ncsbe.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; import { scotusConfig } from "./scrapers/scotus.config.js"; @@ -20,6 +21,7 @@ export const scraperContracts: readonly ScraperEnvContract[] = [ sccCvigConfig, caSosStatementsConfig, ncsbeConfig, + missouriLegislatureConfig, texasCurrentElectionConfig, texasLegislatureConfig, cedarParkCouncilConfig, diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index 2e3ad49..e71db4f 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -5,6 +5,7 @@ import { congress } from "./scrapers/congress.js"; import { durhamBocc } from "./scrapers/durham-bocc.js"; import { durhamOnBase } from "./scrapers/durham-onbase.js"; import { federalregister } from "./scrapers/federalregister.js"; +import { missouriLegislature } from "./scrapers/missouri-legislature.js"; import { ncsbe } from "./scrapers/ncsbe.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; import { scotus } from "./scrapers/scotus.js"; @@ -19,6 +20,7 @@ export const scrapers: readonly Scraper[] = [ sccCvig, caSosStatements, ncsbe, + missouriLegislature, texasCurrentElection, texasLegislature, cedarParkCouncil, diff --git a/apps/scraper/src/scrapers/fixtures/missouri-bill-list.xml b/apps/scraper/src/scrapers/fixtures/missouri-bill-list.xml new file mode 100644 index 0000000..bf5d66a --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/missouri-bill-list.xml @@ -0,0 +1,4 @@ + + HB422026Rhttps://documents.house.mo.gov/xml/261-HB42.xml07-22-2026 10:30:48.253 + HJR72026Rhttps://documents.house.mo.gov/xml/261-HJR7.xml07-21-2026 09:20:10.010 + diff --git a/apps/scraper/src/scrapers/fixtures/missouri-hb42.xml b/apps/scraper/src/scrapers/fixtures/missouri-hb42.xml new file mode 100644 index 0000000..734f73f --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/missouri-hb42.xml @@ -0,0 +1,16 @@ + + HB42HCS HB 42 + <ShortTitle>PUBLIC SAFETY</ShortTitle><LongTitle>Modifies provisions relating to public safety</LongTitle> + 2026-08-2804/03/2026 - Reported Do Pass (H) + https://house.mo.gov/bill.aspx?bill=HB42&year=2026&code=RIntroduced and Read First Time (H)1001002026-01-08 + https://house.mo.gov/bill.aspx?bill=HB42&year=2026&code=RReported Do Pass (H)2002002026-04-031021 + SponsorJamie Example12 + Co-SponsorAlex Example13 + https://documents.house.mo.gov/billtracking/HB42P.pdf1000H.01PPerfected + https://documents.house.mo.gov/billtracking/HB42-summary.pdfPerfected + https://documents.house.mo.gov/billtracking/HB42-fiscal.pdf + https://documents.house.mo.gov/billtracking/HB42-amendment.pdfHA 1 + Crime Prevention and Public Safety2026-02-01 + PUBLIC SAFETY + https://documents.house.mo.gov/billtracking/HB42-witness.pdfPublic testimony + diff --git a/apps/scraper/src/scrapers/fixtures/missouri-senate-actions.xml b/apps/scraper/src/scrapers/fixtures/missouri-senate-actions.xml new file mode 100644 index 0000000..f7b4971 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/missouri-senate-actions.xml @@ -0,0 +1,7 @@ + + SB9HEALTH CAREModifies provisions relating to health care2026-08-28T00:00:00 + Sample, Sam04/10/2026 - Reported Do Pass (H) + https://house.mo.gov/bill.aspx?bill=SB9&year=2026&code=RReported to the House and First Read (H)3001002026-03-20 + Health and Mental Health + https://documents.house.mo.gov/billtracking/SB9-summary.pdfCommittee + diff --git a/apps/scraper/src/scrapers/fixtures/missouri-session-set.js b/apps/scraper/src/scrapers/fixtures/missouri-session-set.js new file mode 100644 index 0000000..192df65 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/missouri-session-set.js @@ -0,0 +1,7 @@ +var sessionyearcode = "261"; +var baseURL = "https://documents.house.mo.gov/xml/261-"; +var specsessionyearcode = "263"; +var specbaseURL = "https://documents.house.mo.gov/xml/263-"; +var specsession2yearcode = "264"; +var specbaseURL2 = "https://documents.house.mo.gov/xml/264-"; +var showSpec = "true"; diff --git a/apps/scraper/src/scrapers/missouri-legislature-parser.ts b/apps/scraper/src/scrapers/missouri-legislature-parser.ts new file mode 100644 index 0000000..c6e5aad --- /dev/null +++ b/apps/scraper/src/scrapers/missouri-legislature-parser.ts @@ -0,0 +1,413 @@ +import { load } from "cheerio"; + +export const MISSOURI_JURISDICTION = + "ocd-jurisdiction/country:us/state:mo/government"; + +export interface MissouriSession { + code: string; + baseUrl: string; + kind: "regular" | "special"; +} + +export interface MissouriBillListEntry { + billNumber: string; + url: string; + sourceVersion: string; + sourceUpdatedAt?: Date; +} + +export type MissouriDocument = { + type: "bill_text" | "analysis" | "fiscal_note"; + description: string; + pdfUrl?: string; +}; + +function clean(value: string | undefined): string | undefined { + const normalized = value?.replace(/\s+/g, " ").trim(); + return normalized || undefined; +} + +function variable(script: string, name: string): string | undefined { + return new RegExp(`\\bvar\\s+${name}\\s*=\\s*['\"]([^'\"]+)['\"]`, "i").exec( + script, + )?.[1]; +} + +export function parseMissouriSessions(script: string): MissouriSession[] { + const code = variable(script, "sessionyearcode"); + const baseUrl = variable(script, "baseURL"); + if (!code || !baseUrl) { + throw new Error("SessionSet.js is missing the active regular session"); + } + const sessions: MissouriSession[] = [{ code, baseUrl, kind: "regular" }]; + if (variable(script, "showSpec")?.toLowerCase() !== "true") return sessions; + for (const [codeName, urlName] of [ + ["specsessionyearcode", "specbaseURL"], + ["specsession2yearcode", "specbaseURL2"], + ] as const) { + const specialCode = variable(script, codeName); + const specialUrl = variable(script, urlName); + if (specialCode && specialUrl && specialCode !== code) { + sessions.push({ + code: specialCode, + baseUrl: specialUrl, + kind: "special", + }); + } + } + return sessions; +} + +export function normalizeMissouriBillNumber(value: string): string { + const match = /^(HCR|HJR|HR|HB|SCR|SJR|SR|SB)\s*0*(\d+)$/i.exec( + value.replace(/\s+/g, ""), + ); + if (!match) + throw new Error(`Unrecognized Missouri bill identifier: ${value}`); + return `${match[1]!.toUpperCase()} ${Number(match[2])}`; +} + +export function parseMissouriTimestamp(value: string): Date | undefined { + const match = + /^(\d{2})-(\d{2})-(\d{4}) (\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?$/.exec( + value.trim(), + ); + if (!match) return undefined; + const [, month, day, year, hour, minute, second, millis = "0"] = match; + const wallClockUtc = Date.UTC( + Number(year), + Number(month) - 1, + Number(day), + Number(hour), + Number(minute), + Number(second), + Number(millis.padEnd(3, "0")), + ); + const guess = new Date(wallClockUtc); + const parts = new Intl.DateTimeFormat("en-US", { + timeZone: "America/Chicago", + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hourCycle: "h23", + }).formatToParts(guess); + const part = (type: Intl.DateTimeFormatPartTypes) => + Number(parts.find((item) => item.type === type)?.value); + const chicagoAsUtc = Date.UTC( + part("year"), + part("month") - 1, + part("day"), + part("hour"), + part("minute"), + part("second"), + Number(millis.padEnd(3, "0")), + ); + return new Date(wallClockUtc - (chicagoAsUtc - guess.valueOf())); +} + +export function parseMissouriBillList(xml: string): MissouriBillListEntry[] { + const $ = load(xml, { xml: true }); + return $("BillXML") + .map((_, element) => { + const row = $(element); + const type = clean(row.find("BillType").first().text()); + const number = clean(row.find("BillNumber").first().text()); + const url = clean(row.find("BillXMLLink").first().text()); + const sourceVersion = clean(row.find("LastTimeRun").first().text()); + if (!type || !number || !url || !sourceVersion) return undefined; + return { + billNumber: normalizeMissouriBillNumber(`${type}${number}`), + url, + sourceVersion, + ...(parseMissouriTimestamp(sourceVersion) && { + sourceUpdatedAt: parseMissouriTimestamp(sourceVersion), + }), + }; + }) + .get() + .filter((entry): entry is MissouriBillListEntry => Boolean(entry)) + .sort((left, right) => left.billNumber.localeCompare(right.billNumber)); +} + +function isoDate(value: string | undefined): string | undefined { + const text = clean(value); + if (!text) return undefined; + const match = /^(\d{1,2})\/(\d{1,2})\/(\d{4})/.exec(text); + if (match) { + return `${match[3]}-${match[1]!.padStart(2, "0")}-${match[2]!.padStart(2, "0")}`; + } + const parsed = new Date(text); + return Number.isNaN(parsed.valueOf()) + ? undefined + : parsed.toISOString().slice(0, 10); +} + +function documents($: ReturnType): MissouriDocument[] { + const result: MissouriDocument[] = []; + const add = ( + type: MissouriDocument["type"], + description: string, + url?: string, + ) => { + const pdfUrl = clean(url); + if (pdfUrl) + result.push({ type, description: clean(description) ?? type, pdfUrl }); + }; + $("BillText").each((_, element) => { + const row = $(element); + add( + "bill_text", + row.find("DocumentName").first().text(), + row.find("BillTextLink").first().text(), + ); + }); + $("BillSummary").each((_, element) => { + const row = $(element); + add( + "analysis", + `Bill summary — ${row.find("DocumentName").first().text()}`, + row.find("SummaryTextLink").first().text(), + ); + }); + $("FiscalNote").each((index, element) => + add( + "fiscal_note", + `Fiscal note ${index + 1}`, + $(element).find("FiscalNoteLink").first().text(), + ), + ); + $("Amendment").each((_, element) => { + const row = $(element); + add( + "analysis", + `Amendment — ${row.find("AmendmentDescription").first().text()}`, + row.find("AmendmentText").first().text(), + ); + }); + add("analysis", "Summary sheet", $("SummarySheetLink").first().text()); + add( + "analysis", + "Governor veto letter", + $("GovernorsVetoLetter").first().text(), + ); + $("Witness").each((index, element) => + add( + "analysis", + `Witness forms — ${ + clean($(element).find("WitnessFormsLinkDescription").first().text()) ?? + index + 1 + }`, + $(element).find("WitnessFormsLink").first().text(), + ), + ); + return result; +} + +export function parseMissouriBill( + xml: string, + options: { + session: string; + sourceVersion: string; + sourceUpdatedAt?: Date; + coverage: "complete_house_export" | "senate_with_house_actions_only"; + }, +) { + const $ = load(xml, { xml: true }); + const root = $("BillInformation").first(); + const billNumber = normalizeMissouriBillNumber( + root.find("BillNumber").first().text(), + ); + const chamber = billNumber.startsWith("H") + ? ("House" as const) + : ("Senate" as const); + const title = + clean(root.find("Title > LongTitle").first().text()) ?? + clean(root.children("LongTitle").first().text()) ?? + clean(root.find("Title > ShortTitle").first().text()) ?? + clean(root.children("ShortTitle").first().text()); + if (!title) throw new Error(`${billNumber} is missing a title`); + const shortTitle = + clean(root.find("Title > ShortTitle").first().text()) ?? + clean(root.children("ShortTitle").first().text()); + + const sponsorships: { + name: string; + classification: "primary" | "cosponsor"; + chamber: "House" | "Senate"; + }[] = []; + root.children("Sponsor").each((_, element) => { + const row = $(element); + const name = clean(row.find("FullName").first().text()); + if (!name) return; + sponsorships.push({ + name, + classification: /co/i.test(row.find("SponsorType").first().text()) + ? "cosponsor" + : "primary", + chamber, + }); + }); + const senateSponsor = clean(root.children("SponsorName").first().text()); + if (senateSponsor) + sponsorships.push({ + name: senateSponsor, + classification: "primary", + chamber: "Senate", + }); + + const actions: { date: string; text: string; type?: string }[] = []; + const votes: { + identifier: string; + date?: string; + chamber?: "House" | "Senate"; + motion?: string; + counts: { option: string; value: number }[]; + votes: never[]; + }[] = []; + root.children("Action").each((_, element) => { + const row = $(element); + const date = isoDate(row.find("PubDate").first().text()); + const text = clean(row.find("Description").first().text()); + if (!date || !text) return; + const guid = clean(row.find("Guid").first().text()); + actions.push({ date, text, ...(guid && { type: `guid:${guid}` }) }); + const rollCall = row.find("RollCall").first(); + if (rollCall.length) { + const count = (selector: string) => + Number(clean(rollCall.find(selector).first().text())); + const counts = [ + ["Yes", count("TotalYes")], + ["No", count("TotalNo")], + ["Present", count("TotalPresent")], + ] as const; + votes.push({ + identifier: guid ?? `${date}:${text}`, + date, + chamber: /\(S\)/.test(text) ? "Senate" : "House", + motion: text, + counts: counts + .filter((entry) => Number.isInteger(entry[1])) + .map(([option, value]) => ({ option, value })), + votes: [], + }); + } + }); + const effectiveDateText = isoDate( + root.find("ProposedEffectiveDate").first().text(), + ); + const sponsor = sponsorships + .filter((item) => item.classification === "primary") + .map((item) => item.name) + .join(" | ") + .slice(0, 256); + root.find("Hearings").each((_, element) => { + const hearing = $(element); + const name = clean(hearing.find("CommitteeName, CommName").first().text()); + const date = + isoDate(hearing.find("NoticeDate").first().text()) ?? actions[0]?.date; + if (name && date) { + actions.push({ + date, + text: `Committee hearing: ${name}`, + type: "committee", + }); + } + }); + if (effectiveDateText) { + actions.push({ + date: effectiveDateText, + text: "Proposed effective date", + type: "effective_date", + }); + } + actions.sort((left, right) => left.date.localeCompare(right.date)); + const url = + clean(root.children("Action").first().find("Link").first().text()) ?? + `https://house.mo.gov/bill.aspx?bill=${billNumber.replace(/\s+/g, "")}&year=20${options.session.slice(0, 2)}&code=${options.session.slice(2) === "1" ? "R" : `S${options.session.slice(2)}`}`; + return { + billNumber, + title, + ...(shortTitle && { description: shortTitle }), + ...(sponsor && { sponsor }), + ...(clean(root.find("LastAction").first().text()) && { + status: clean(root.find("LastAction").first().text())!.slice(0, 100), + }), + ...(actions[0]?.date && { + introducedDate: new Date(`${actions[0].date}T12:00:00Z`), + }), + chamber, + jurisdiction: MISSOURI_JURISDICTION, + legislativeSession: options.session, + subjects: root + .find("SubjectIndex > SubjectName") + .map((_, element) => clean($(element).text())) + .get() + .filter(Boolean) as string[], + sponsorships, + documents: documents($), + votes, + actions, + versions: [ + { + hash: options.sourceVersion, + updatedAt: + options.sourceUpdatedAt?.toISOString() ?? + `${actions.at(-1)?.date ?? "1970-01-01"}T00:00:00.000Z`, + changes: options.coverage, + }, + ], + url, + }; +} + +export function missouriCommitteesFromActions( + actions: readonly { text: string; type?: string }[], +): string[] { + return [ + ...new Set( + actions.flatMap((action) => { + if (action.type === "committee") { + const name = clean( + action.text.replace(/^Committee hearing:\s*/i, ""), + ); + return name ? [name] : []; + } + const match = + /(?:Re-referred to Committee|Referred):\s*(.+?)(?:\([HS]\))?$/i.exec( + action.text, + ); + const name = clean(match?.[1]); + return name ? [name] : []; + }), + ), + ]; +} + +export function missouriEffectiveDateFromActions( + actions: readonly { date: string; type?: string }[], +): string | undefined { + return actions.find((action) => action.type === "effective_date")?.date; +} + +export function parseMissouriSenateActionList( + xml: string, + session: string, + sourceVersion: string, + sourceUpdatedAt?: Date, +) { + const $ = load(xml, { xml: true }); + return $("SenateBillsWithHouseActions > BillInformation") + .map((_, element) => + parseMissouriBill($.xml(element), { + session, + sourceVersion, + ...(sourceUpdatedAt && { sourceUpdatedAt }), + coverage: "senate_with_house_actions_only", + }), + ) + .get() + .filter((bill) => bill.chamber === "Senate"); +} diff --git a/apps/scraper/src/scrapers/missouri-legislature-source.ts b/apps/scraper/src/scrapers/missouri-legislature-source.ts new file mode 100644 index 0000000..498161d --- /dev/null +++ b/apps/scraper/src/scrapers/missouri-legislature-source.ts @@ -0,0 +1,6 @@ +export function missouriRefreshExpiresAt( + now: Date, + intervalMs = 30 * 60 * 1000, +): Date { + return new Date(now.valueOf() + intervalMs); +} diff --git a/apps/scraper/src/scrapers/missouri-legislature.config.ts b/apps/scraper/src/scrapers/missouri-legislature.config.ts new file mode 100644 index 0000000..c79ff53 --- /dev/null +++ b/apps/scraper/src/scrapers/missouri-legislature.config.ts @@ -0,0 +1,12 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const missouriLegislatureConfig = { + id: "missouri-legislature", + name: "Missouri General Assembly", + source: + "Missouri House official current-session BillList.XML, bill XML, and partial SenateActList.XML", + environment: { + required: ["POSTGRES_URL"], + optional: ["OPEN_STATES_API_KEY", "MISSOURI_LEGISLATURE_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/missouri-legislature.test.ts b/apps/scraper/src/scrapers/missouri-legislature.test.ts new file mode 100644 index 0000000..c2e1803 --- /dev/null +++ b/apps/scraper/src/scrapers/missouri-legislature.test.ts @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { CreateBillSchema } from "@acme/db/schema"; + +import { + missouriCommitteesFromActions, + missouriEffectiveDateFromActions, + parseMissouriBill, + parseMissouriBillList, + parseMissouriSenateActionList, + parseMissouriSessions, +} from "./missouri-legislature-parser.js"; +import { missouriRefreshExpiresAt } from "./missouri-legislature-source.js"; + +const fixture = (name: string) => + readFile(new URL(`./fixtures/${name}`, import.meta.url), "utf8"); + +void test("uses an exact 30-minute durable refresh expiration", () => { + assert.equal( + missouriRefreshExpiresAt( + new Date("2026-07-22T18:00:00.000Z"), + ).toISOString(), + "2026-07-22T18:30:00.000Z", + ); +}); + +void test("discovers enabled regular and special sessions from SessionSet.js", async () => { + assert.deepEqual( + parseMissouriSessions(await fixture("missouri-session-set.js")), + [ + { + code: "261", + baseUrl: "https://documents.house.mo.gov/xml/261-", + kind: "regular", + }, + { + code: "263", + baseUrl: "https://documents.house.mo.gov/xml/263-", + kind: "special", + }, + { + code: "264", + baseUrl: "https://documents.house.mo.gov/xml/264-", + kind: "special", + }, + ], + ); +}); + +void test("parses stable bill-list versions for changed-row filtering", async () => { + const entries = parseMissouriBillList( + await fixture("missouri-bill-list.xml"), + ); + assert.equal(entries[0]?.billNumber, "HB 42"); + assert.equal(entries[0]?.sourceVersion, "07-22-2026 10:30:48.253"); + assert.equal( + entries[0]?.sourceUpdatedAt?.toISOString(), + "2026-07-22T15:30:48.253Z", + ); + const persisted = new Map([["HB 42", entries[0]!.sourceVersion]]); + assert.deepEqual( + entries + .filter( + (entry) => persisted.get(entry.billNumber) !== entry.sourceVersion, + ) + .map((entry) => entry.billNumber), + ["HJR 7"], + ); +}); + +void test("normalizes Missouri sponsors, actions, committees, dates, votes, and documents", async () => { + const bill = parseMissouriBill(await fixture("missouri-hb42.xml"), { + session: "261", + sourceVersion: "07-22-2026 10:30:48.253", + sourceUpdatedAt: new Date("2026-07-22T15:30:48.253Z"), + coverage: "complete_house_export", + }); + assert.equal(bill.billNumber, "HB 42"); + assert.equal(bill.sponsor, "Jamie Example"); + assert.equal(missouriEffectiveDateFromActions(bill.actions), "2026-08-28"); + assert.deepEqual(missouriCommitteesFromActions(bill.actions), [ + "Crime Prevention and Public Safety", + ]); + assert.deepEqual(bill.votes[0]?.counts, [ + { option: "Yes", value: 10 }, + { option: "No", value: 2 }, + { option: "Present", value: 1 }, + ]); + assert.deepEqual( + bill.documents.map((document) => document.type), + ["bill_text", "analysis", "fiscal_note", "analysis", "analysis"], + ); + assert.equal( + CreateBillSchema.safeParse({ + ...bill, + sourceWebsite: "documents.house.mo.gov", + }).success, + true, + ); +}); + +void test("labels SenateActList rows as House-actions-only coverage", async () => { + const [bill] = parseMissouriSenateActionList( + await fixture("missouri-senate-actions.xml"), + "261", + "sha256:fixture", + new Date("2026-07-22T18:00:00.000Z"), + ); + assert.equal(bill?.billNumber, "SB 9"); + assert.equal(bill?.chamber, "Senate"); + assert.equal(bill?.versions[0]?.changes, "senate_with_house_actions_only"); + assert.equal(bill?.versions[0]?.updatedAt, "2026-07-22T18:00:00.000Z"); + assert.deepEqual(missouriCommitteesFromActions(bill?.actions ?? []), [ + "Health and Mental Health", + ]); +}); diff --git a/apps/scraper/src/scrapers/missouri-legislature.ts b/apps/scraper/src/scrapers/missouri-legislature.ts new file mode 100644 index 0000000..7deef3f --- /dev/null +++ b/apps/scraper/src/scrapers/missouri-legislature.ts @@ -0,0 +1,280 @@ +import pLimit from "p-limit"; + +import { and, eq, inArray, lte, notInArray } from "@acme/db"; +import { db } from "@acme/db/client"; +import { Bill, CivicApiCache } from "@acme/db/schema"; + +import type { BillData, Scraper } from "../utils/types.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { upsertContent } from "../utils/db/operations.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createContentHash } from "../utils/hash.js"; +import { createLogger } from "../utils/log.js"; +import { + MISSOURI_JURISDICTION, + parseMissouriBill, + parseMissouriBillList, + parseMissouriSenateActionList, + parseMissouriSessions, +} from "./missouri-legislature-parser.js"; +import { missouriRefreshExpiresAt } from "./missouri-legislature-source.js"; +import { missouriLegislatureConfig } from "./missouri-legislature.config.js"; + +const logger = createLogger("missouri-legislature"); +const SESSION_DESCRIPTOR = "https://documents.house.mo.gov/SessionSet.js"; +const SOURCE_WEBSITE = "documents.house.mo.gov"; +const REFRESH_SOURCE = "missouri-house-xml"; +const REFRESH_ADDRESS_HASH = createContentHash(REFRESH_SOURCE); +const REFRESH_PARAMS = "{}"; +const REFRESH_INTERVAL_MS = 30 * 60 * 1000; +const FETCH_CONCURRENCY = 2; + +async function claimRefresh(now = new Date()): Promise { + const expiresAt = missouriRefreshExpiresAt(now, REFRESH_INTERVAL_MS); + const rows = await db + .insert(CivicApiCache) + .values({ + addressHash: REFRESH_ADDRESS_HASH, + endpoint: REFRESH_SOURCE, + params: REFRESH_PARAMS, + responseData: { source: REFRESH_SOURCE }, + fetchedAt: now, + expiresAt, + }) + .onConflictDoUpdate({ + target: [ + CivicApiCache.addressHash, + CivicApiCache.endpoint, + CivicApiCache.params, + ], + set: { fetchedAt: now, expiresAt }, + setWhere: lte(CivicApiCache.expiresAt, now), + }) + .returning({ id: CivicApiCache.id }); + return rows.length > 0; +} + +async function text(url: string): Promise { + return (await fetchWithRetry(url, { timeoutMs: 30_000 })).text(); +} + +function openStatesSession(session: string): string { + const year = `20${session.slice(0, 2)}`; + const suffix = Number(session.slice(2)); + return suffix <= 1 ? year : `${year}S${suffix - 2}`; +} + +interface OpenStatesSearchResponse { + results?: { id: string; identifier: string; session: string }[]; +} + +export async function matchMissouriOpenStatesBillId( + session: string, + billNumber: string, + apiKey = process.env.OPEN_STATES_API_KEY, +): Promise { + if (!apiKey) return undefined; + const url = new URL("https://v3.openstates.org/bills"); + url.searchParams.set("jurisdiction", MISSOURI_JURISDICTION); + url.searchParams.set("session", openStatesSession(session)); + url.searchParams.set("q", billNumber); + url.searchParams.set("per_page", "5"); + const response = await fetch(url, { + headers: { Accept: "application/json", "X-API-KEY": apiKey }, + }); + if (!response.ok) return undefined; + const payload = (await response.json()) as OpenStatesSearchResponse; + return payload.results?.find( + (bill) => + bill.session === openStatesSession(session) && + bill.identifier.replace(/\s+/g, "").toUpperCase() === + billNumber.replace(/\s+/g, "").toUpperCase(), + )?.id; +} + +async function persistBill(data: BillData): Promise { + if (!data.legislativeSession) { + throw new Error(`${data.billNumber} is missing its Missouri session`); + } + let openStatesId: string | undefined; + try { + openStatesId = await matchMissouriOpenStatesBillId( + data.legislativeSession, + data.billNumber, + ); + } catch (error) { + logger.debug( + `Optional Open States match skipped for ${data.billNumber}: ${error instanceof Error ? error.message : error}`, + ); + } + await upsertContent( + { + type: "bill", + data: { + ...data, + ...(openStatesId && { openStatesId }), + sourceWebsite: SOURCE_WEBSITE, + }, + }, + { skipEnrichment: true }, + ); +} + +function withVersionHistory( + bill: ReturnType, + histories: Map>, +): BillData { + const key = `${bill.legislativeSession}:${bill.billNumber}`; + const previous = histories.get(key) ?? []; + const current = bill.versions[0]!; + const versions = + previous.at(-1)?.hash === current.hash + ? previous + : [...previous, current].slice(-50); + histories.set(key, versions); + return { ...bill, versions, sourceWebsite: SOURCE_WEBSITE }; +} + +export async function scrapeMissouriLegislature(options: { + maxItems: number; + now?: Date; +}): Promise { + const refreshNow = options.now ?? new Date(); + if (!(await claimRefresh(refreshNow))) { + logger.info( + "Skipping Missouri XML refresh: the official 30-minute polling interval has not elapsed.", + ); + return; + } + + const descriptor = await text(SESSION_DESCRIPTOR); + const sessions = parseMissouriSessions(descriptor); + const activeSessionCodes = sessions.map((session) => session.code); + await db + .delete(Bill) + .where( + and( + eq(Bill.jurisdiction, MISSOURI_JURISDICTION), + eq(Bill.sourceWebsite, SOURCE_WEBSITE), + notInArray(Bill.legislativeSession, activeSessionCodes), + ), + ); + const existing = await db + .select({ + billNumber: Bill.billNumber, + session: Bill.legislativeSession, + versions: Bill.versions, + }) + .from(Bill) + .where( + and( + eq(Bill.jurisdiction, MISSOURI_JURISDICTION), + eq(Bill.sourceWebsite, SOURCE_WEBSITE), + inArray(Bill.legislativeSession, activeSessionCodes), + ), + ); + const versionHistories = new Map( + existing.map((row) => [ + `${row.session}:${row.billNumber}`, + row.versions ?? [], + ]), + ); + const currentVersion = (key: string) => + versionHistories.get(key)?.at(-1)?.hash; + const limit = pLimit(FETCH_CONCURRENCY); + const pending: BillData[] = []; + + for (const [sessionIndex, session] of sessions.entries()) { + const sessionBudget = + Math.floor(options.maxItems / sessions.length) + + (sessionIndex < options.maxItems % sessions.length ? 1 : 0); + const sessionEnd = pending.length + sessionBudget; + const billListXml = await text(`${session.baseUrl}BillList.XML`); + const changed = parseMissouriBillList(billListXml) + .filter( + (entry) => + currentVersion(`${session.code}:${entry.billNumber}`) !== + entry.sourceVersion, + ) + .sort((left, right) => { + const leftMissing = !versionHistories.has( + `${session.code}:${left.billNumber}`, + ); + const rightMissing = !versionHistories.has( + `${session.code}:${right.billNumber}`, + ); + return Number(rightMissing) - Number(leftMissing); + }); + const senateReserve = Math.min( + sessionBudget, + Math.max(1, Math.floor(sessionBudget * 0.2)), + ); + const remaining = Math.max(0, sessionEnd - pending.length - senateReserve); + const houseBills = await Promise.all( + changed.slice(0, remaining).map((entry) => + limit(async () => + parseMissouriBill(await text(entry.url), { + session: session.code, + sourceVersion: entry.sourceVersion, + ...(entry.sourceUpdatedAt && { + sourceUpdatedAt: entry.sourceUpdatedAt, + }), + coverage: "complete_house_export", + }), + ), + ), + ); + pending.push( + ...houseBills.map((bill) => withVersionHistory(bill, versionHistories)), + ); + + if (pending.length < sessionEnd) { + const senateXml = await text(`${session.baseUrl}SenateActList.XML`); + const senateVersion = `sha256:${createContentHash(senateXml)}`; + const senateBills = parseMissouriSenateActionList( + senateXml, + session.code, + senateVersion, + refreshNow, + ) + .filter( + (bill) => + currentVersion(`${session.code}:${bill.billNumber}`) !== + senateVersion, + ) + .sort((left, right) => { + const leftMissing = !versionHistories.has( + `${session.code}:${left.billNumber}`, + ); + const rightMissing = !versionHistories.has( + `${session.code}:${right.billNumber}`, + ); + return Number(rightMissing) - Number(leftMissing); + }); + pending.push( + ...senateBills + .slice(0, sessionEnd - pending.length) + .map((bill) => withVersionHistory(bill, versionHistories)), + ); + } + } + + setExpectedTotal(pending.length); + for (const bill of pending) await persistBill(bill); + logger.success( + `Persisted ${pending.length} changed Missouri bills across active session(s) ${activeSessionCodes.join(", ")}. Senate rows include House actions only.`, + ); +} + +async function scrape(options?: { maxItems?: number }): Promise { + await scrapeMissouriLegislature({ + maxItems: + options?.maxItems ?? + (Number(process.env.MISSOURI_LEGISLATURE_MAX_ITEMS) || 100), + }); +} + +export const missouriLegislature: Scraper = { + ...missouriLegislatureConfig, + scrape, +}; diff --git a/apps/scraper/src/utils/db/operations.ts b/apps/scraper/src/utils/db/operations.ts index 3ab5d39..e2385e2 100644 --- a/apps/scraper/src/utils/db/operations.ts +++ b/apps/scraper/src/utils/db/operations.ts @@ -78,6 +78,7 @@ function hashFields(input: ContentData): string { documents: input.data.documents, votes: input.data.votes, actions: input.data.actions, + versions: input.data.versions, }); case "government_content": return JSON.stringify({ @@ -241,14 +242,10 @@ export async function upsertContent( ...d, description: preGeneratedDescription || d.description, contentHash: newContentHash, - versions: [], + versions: d.versions ?? [], }) .onConflictDoUpdate({ - target: [ - Bill.billNumber, - Bill.sourceWebsite, - Bill.legislativeSession, - ], + target: [Bill.billNumber, Bill.sourceWebsite, Bill.legislativeSession], set: { title: d.title, description: d.description, @@ -267,6 +264,7 @@ export async function upsertContent( summary: d.summary, fullText: d.fullText, actions: d.actions, + ...(d.versions && { versions: d.versions }), url: d.url, contentHash: newContentHash, updatedAt: new Date(), diff --git a/docs/scraper.md b/docs/scraper.md index 7111888..adef7bc 100644 --- a/docs/scraper.md +++ b/docs/scraper.md @@ -27,6 +27,7 @@ process environment at runtime, not embedded during the build. | `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | | `texas-current-election.ts` | Texas SOS + TLC | `election_source_snapshot` | current-cycle JSON + deterministic PDF text parsing | | `texas-legislature.ts` | Texas Legislative Council FTP | `bill` | current-session XML + bulk HTML; no site mining | +| `missouri-legislature.ts` | Missouri House XML exports | `bill` | active sessions; Senate coverage is House-actions-only | | `civicengage.ts` | Cedar Park official council page | local-government tables | CivicEngage entry page + Municode embed; deterministic HTML/PDF parsing | | `durham-onbase.ts` | Durham OnBase Agenda Online | local-government tables | current-cycle meetings, items, attachments, and official actions | | `durham-bocc.ts` | Durham County Legistar API | local-government tables | current-cycle meetings, items, actions, votes, and documents | @@ -81,6 +82,18 @@ votes, and an optional exact Open States ID. The scraper selects only the latest FTP session and sets `skipEnrichment`; the API exposes only that newest session through `content.texasBills`, with full supporting material in `content.getById`. +Missouri uses the same rows and `content.stateBills` reader. Active session +codes come from the official +[`SessionSet.js`](https://documents.house.mo.gov/SessionSet.js); +[`BillList.XML` fields](https://documents.house.mo.gov/XMLBillList.html), +including `LastTimeRun`, drive +individual XML change detection. A `CivicApiCache` refresh lease enforces the +official 30-minute minimum interval, and fetch concurrency is capped at two. +Senate rows come only from `SenateActList.XML` and retain the explicit +`senate_with_house_actions_only` coverage marker in `bill.versions`. +The [official export guidance](https://documents.house.mo.gov/) documents the +hourly generation schedule and the 30-minute minimum polling interval. + ## AI Pipeline Provider config lives in `apps/scraper/src/utils/ai/provider.ts`: text uses **OpenRouter** first, then an OpenAI-compatible local endpoint (`LOCAL_LLM_BASE_URL`, such as Ollama), with direct DeepSeek retained only as a deprecated last resort. PDF vision fallback uses **Gemini `gemini-2.5-flash`**. Images use hosted **Black Forest Labs FLUX.2 Klein 9B**, then `LOCAL_FLUX_BASE_URL` as a local fallback. Provider usage and hosted-image costs are tracked per run. diff --git a/packages/api/src/lib/state-legislation.test.ts b/packages/api/src/lib/state-legislation.test.ts new file mode 100644 index 0000000..07da303 --- /dev/null +++ b/packages/api/src/lib/state-legislation.test.ts @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { stateBillMetadata } from "./state-legislation"; + +void test("derives state metadata from shared bill actions and versions", () => { + assert.deepEqual( + stateBillMetadata( + [ + { + date: "2026-02-01", + text: "Committee hearing: Public Safety", + type: "committee", + }, + { date: "2026-02-02", text: "Referred: Rules - Legislative(H)" }, + { + date: "2026-08-28", + text: "Proposed effective date", + type: "effective_date", + }, + ], + [ + { + hash: "07-22-2026 10:30:48.253", + updatedAt: "2026-07-22T10:30:48.253Z", + changes: "complete_house_export", + }, + ], + ), + { + committees: ["Public Safety", "Rules - Legislative"], + effectiveDate: "2026-08-28", + sourceVersion: "07-22-2026 10:30:48.253", + sourceUpdatedAt: "2026-07-22T10:30:48.253Z", + sourceCoverage: "complete_house_export", + }, + ); +}); diff --git a/packages/api/src/lib/state-legislation.ts b/packages/api/src/lib/state-legislation.ts new file mode 100644 index 0000000..7c7979e --- /dev/null +++ b/packages/api/src/lib/state-legislation.ts @@ -0,0 +1,43 @@ +export interface StateBillAction { + date: string; + text: string; + type?: string; +} + +export interface StateBillVersion { + hash: string; + updatedAt: string; + changes: string; +} + +export function stateBillMetadata( + actions: readonly StateBillAction[], + versions: readonly StateBillVersion[], +) { + const source = versions.at(-1); + const committees = [ + ...new Set( + actions.flatMap((action) => { + if (action.type === "committee") { + const name = action.text + .replace(/^Committee hearing:\s*/i, "") + .trim(); + return name ? [name] : []; + } + const match = + /(?:Re-referred to Committee|Referred):\s*(.+?)(?:\([HS]\))?$/i.exec( + action.text, + ); + return match?.[1]?.trim() ? [match[1].trim()] : []; + }), + ), + ]; + return { + committees, + effectiveDate: actions.find((action) => action.type === "effective_date") + ?.date, + sourceVersion: source?.hash, + sourceUpdatedAt: source?.updatedAt, + sourceCoverage: source?.changes, + }; +} diff --git a/packages/api/src/router/content.ts b/packages/api/src/router/content.ts index 102ba20..8b85b01 100644 --- a/packages/api/src/router/content.ts +++ b/packages/api/src/router/content.ts @@ -13,6 +13,7 @@ import { } from "@acme/db/schema"; import { parseBillSponsor, sponsorRole } from "../lib/bill-sponsor"; +import { stateBillMetadata } from "../lib/state-legislation"; import { protectedProcedure, publicProcedure } from "../trpc"; const SAVED_CONTENT_TYPES = [ @@ -22,6 +23,83 @@ const SAVED_CONTENT_TYPES = [ ] as const; type SavedContentType = (typeof SAVED_CONTENT_TYPES)[number]; +const STATE_JURISDICTIONS = { + MO: "ocd-jurisdiction/country:us/state:mo/government", + TX: "ocd-jurisdiction/country:us/state:tx/government", +} as const; +const STATE_SOURCES = { + MO: "documents.house.mo.gov", + TX: "capitol.texas.gov", +} as const; + +async function listCurrentStateBills(input: { + stateCode: keyof typeof STATE_JURISDICTIONS; + limit: number; + cursor: number; +}) { + const jurisdiction = STATE_JURISDICTIONS[input.stateCode]; + const sourceWebsite = STATE_SOURCES[input.stateCode]; + let legislativeSession: string | undefined; + if (input.stateCode === "TX") { + const [latest] = await db + .select({ legislativeSession: Bill.legislativeSession }) + .from(Bill) + .where( + and( + eq(Bill.jurisdiction, jurisdiction), + eq(Bill.sourceWebsite, sourceWebsite), + ), + ) + .orderBy(desc(Bill.updatedAt), desc(Bill.createdAt)) + .limit(1); + legislativeSession = latest?.legislativeSession; + if (!legislativeSession) return { items: [], nextCursor: undefined }; + } + const rows = await db + .select({ + id: Bill.id, + billNumber: Bill.billNumber, + title: Bill.title, + description: Bill.description, + summary: Bill.summary, + status: Bill.status, + chamber: Bill.chamber, + legislativeSession: Bill.legislativeSession, + openStatesId: Bill.openStatesId, + subjects: Bill.subjects, + actions: Bill.actions, + versions: Bill.versions, + updatedAt: Bill.updatedAt, + }) + .from(Bill) + .where( + legislativeSession + ? and( + eq(Bill.jurisdiction, jurisdiction), + eq(Bill.sourceWebsite, sourceWebsite), + eq(Bill.legislativeSession, legislativeSession), + ) + : and( + eq(Bill.jurisdiction, jurisdiction), + eq(Bill.sourceWebsite, sourceWebsite), + ), + ) + .orderBy(desc(Bill.updatedAt), desc(Bill.billNumber)) + .limit(input.limit + 1) + .offset(input.cursor); + const hasMore = rows.length > input.limit; + const items = (hasMore ? rows.slice(0, input.limit) : rows).map( + ({ actions, versions, ...row }) => ({ + ...row, + ...stateBillMetadata(actions ?? [], versions ?? []), + }), + ); + return { + items, + nextCursor: hasMore ? input.cursor + input.limit : undefined, + }; +} + interface ContentImageRef { id: string; type: SavedContentType | "general"; @@ -185,6 +263,16 @@ const _ContentDetailSchema = ContentCardSchema.extend({ export type ContentDetail = z.infer; export const contentRouter = { + stateBills: publicProcedure + .input( + z.object({ + stateCode: z.enum(["MO", "TX"]), + limit: z.number().int().min(1).max(50).default(20), + cursor: z.number().int().min(0).default(0), + }), + ) + .query(({ input }) => listCurrentStateBills(input)), + // Read the latest Texas bulk session persisted by the scraper. This route is // intentionally current-session only; it is not a historical session browser. texasBills: publicProcedure @@ -196,47 +284,13 @@ export const contentRouter = { }) .optional(), ) - .query(async ({ input }) => { - const jurisdiction = "ocd-jurisdiction/country:us/state:tx/government"; - const [latest] = await db - .select({ legislativeSession: Bill.legislativeSession }) - .from(Bill) - .where(eq(Bill.jurisdiction, jurisdiction)) - .orderBy(desc(Bill.updatedAt), desc(Bill.createdAt)) - .limit(1); - if (!latest) return { items: [], nextCursor: undefined }; - const limit = input?.limit ?? 20; - const cursor = input?.cursor ?? 0; - const rows = await db - .select({ - id: Bill.id, - billNumber: Bill.billNumber, - title: Bill.title, - description: Bill.description, - summary: Bill.summary, - status: Bill.status, - chamber: Bill.chamber, - legislativeSession: Bill.legislativeSession, - openStatesId: Bill.openStatesId, - subjects: Bill.subjects, - updatedAt: Bill.updatedAt, - }) - .from(Bill) - .where( - and( - eq(Bill.jurisdiction, jurisdiction), - eq(Bill.legislativeSession, latest.legislativeSession), - ), - ) - .orderBy(desc(Bill.updatedAt), desc(Bill.billNumber)) - .limit(limit + 1) - .offset(cursor); - const hasMore = rows.length > limit; - return { - items: hasMore ? rows.slice(0, limit) : rows, - nextCursor: hasMore ? cursor + limit : undefined, - }; - }), + .query(({ input }) => + listCurrentStateBills({ + stateCode: "TX", + limit: input?.limit ?? 20, + cursor: input?.cursor ?? 0, + }), + ), // Get all content from database getAll: publicProcedure.query(async () => { @@ -643,6 +697,7 @@ export const contentRouter = { jurisdiction: b.jurisdiction, legislativeSession: b.legislativeSession || undefined, openStatesId: b.openStatesId ?? undefined, + ...stateBillMetadata(b.actions ?? [], b.versions ?? []), subjects: b.subjects ?? [], sponsorships: b.sponsorships ?? [], documents: b.documents ?? [], diff --git a/packages/env/src/registry.ts b/packages/env/src/registry.ts index 472daf5..28008a0 100644 --- a/packages/env/src/registry.ts +++ b/packages/env/src/registry.ts @@ -92,6 +92,11 @@ const scraperSourceLimitDefinitions = [ "Texas Legislature bills per current-session run.", "100", ], + [ + "MISSOURI_LEGISLATURE_MAX_ITEMS", + "Missouri General Assembly changed bills per current-session run.", + "100", + ], [ "CEDAR_PARK_COUNCIL_MAX_ITEMS", "Cedar Park City Council meetings per run.",