diff --git a/apps/scraper/README.md b/apps/scraper/README.md index 2e614c7..94841c6 100644 --- a/apps/scraper/README.md +++ b/apps/scraper/README.md @@ -4,7 +4,7 @@ Pulls in government content like bills, court cases, and White House content and ## Active data sources -Only these five are registered and run by `all`: +Only these six are registered and run by `all`: | CLI name | Source and data fetched | Stored/used as | | ------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | @@ -13,6 +13,7 @@ Only these five are registered and run by `all`: | `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | | `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | | `ca-sos-statements` | California SOS statewide-office candidate-statement pages | Candidate statements in `CivicApiCache`; the API reads the cache and can fall back to the live source | +| `texas-legislature` | Texas Legislative Council anonymous FTP: current-session history XML and bulk documents | State-aware `bill` rows; read through `content.texasBills` and `content.getById` | `vote411`, `ca-lao-fiscal`, and `ca-vig-archive` remain under `src/scrapers/disabled/` and do not run. Their caches had no application @@ -55,6 +56,7 @@ work: | `COURTLISTENER_API_KEY` | Optional | Higher CourtListener limits for `scotus`. | | `GOOGLE_API_KEY` / `GOOGLE_SEARCH_ENGINE_ID` | Optional pair | Google Custom Search article thumbnails. | | `GOOGLE_GENERATIVE_AI_API_KEY` | Optional | Gemini vision fallback for `scc-cvig` PDF extraction. | +| `OPEN_STATES_API_KEY` | Optional | Adds an exact Open States bill ID when Texas jurisdiction/session/identifier match. | See [the launch environment guide](../../docs/launch.md) for the complete per-scraper matrix, provider setup links, defaults, and production guidance. @@ -124,6 +126,7 @@ CONGRESS_MAX_ITEMS=10 pnpm --filter @acme/scraper run start congress | `SCOTUS_MAX_ITEMS` | 50 | CourtListener opinion clusters | | `SCC_CVIG_MAX_ITEMS` | 10 | Voter-guide PDF documents | | `CA_SOS_MAX_ITEMS` | 9 | Statewide-office candidate-statement pages | +| `TEXAS_LEGISLATURE_MAX_ITEMS` | 100 | Bills from the latest Texas bulk session | | `SCRAPER_MAX_NEW_ITEMS_PER_RUN` | 10 | New records receiving expensive AI/image enrichment | These are per-run limits, not durable calendar-day quotas. Schedule one run per @@ -152,6 +155,34 @@ await scrapeCongress({ --- +## Texas Legislature Online (`texas-legislature.ts`) + +Uses only the Texas Legislative Council's anonymous FTP service at +`ftp.legis.state.tx.us`; it does not crawl interactive TLO bill pages. The job: + +- discovers the newest session directory under `/bills` and rejects a stale + `TEXAS_LEGISLATURE_SESSION` assertion (official codes look like `89R` or + `892`); +- parses bill-history XML for identity, caption, sponsors, subjects, actions, + structured votes when present, and document metadata; +- downloads bill text, analyses, and fiscal notes from the matching FTP bulk + HTML paths and stores their extracted text alongside official HTML/PDF links; +- optionally stores an exact Open States ID without using Open States as the + legislative data source. + +Run a small current-session import: + +```bash +pnpm --filter @acme/scraper run start texas-legislature --max-items 10 +``` + +Apply `packages/db/migrations/add_state_legislation_fields.sql` before the first +run. The public `content.texasBills` procedure lists only the newest persisted +Texas session; `content.getById` returns its documents, actions, and votes. This +work deliberately does not provide historical-session browsing or backfills. + +--- + ## Court cases (`scotus.ts`) Uses the [CourtListener API](https://www.courtlistener.com/api/) — free, works without a key. Fetches recent opinions and pulls in the plain-text opinion content for AI article generation. @@ -179,4 +210,7 @@ All scrapers call into `src/utils/db/operations.ts`. Each time a bill or case is - If the **content changed** → regenerates the article - If **nothing changed** → backfills any missing AI summary/article/thumbnail fields, otherwise skips AI generation +The Texas importer passes `skipEnrichment` so official source data is persisted +without AI summaries, generated imagery, or videos. + Set `SCRAPER_FORCE_AI_REGEN=1` to force a full AI refresh even when the record already has AI content. diff --git a/apps/scraper/package.json b/apps/scraper/package.json index 3cf5819..75baf44 100644 --- a/apps/scraper/package.json +++ b/apps/scraper/package.json @@ -12,6 +12,7 @@ "@ai-sdk/google": "^3.0.53", "@openrouter/ai-sdk-provider": "^2.10.0", "ai": "^6.0.141", + "basic-ftp": "^6.0.1", "cheerio": "^1.2.0", "consola": "^3.4.2", "drizzle-orm": "^0.45.2", diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index bfcfd04..4c061b1 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -5,6 +5,7 @@ import { congressConfig } from "./scrapers/congress.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; import { scotusConfig } from "./scrapers/scotus.config.js"; +import { texasLegislatureConfig } from "./scrapers/texas-legislature.config.js"; export const scraperContracts: readonly ScraperEnvContract[] = [ federalregisterConfig, @@ -12,4 +13,5 @@ export const scraperContracts: readonly ScraperEnvContract[] = [ scotusConfig, sccCvigConfig, caSosStatementsConfig, + texasLegislatureConfig, ]; diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index d4ab8d4..c3b728a 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -4,6 +4,7 @@ import { congress } from "./scrapers/congress.js"; import { federalregister } from "./scrapers/federalregister.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; import { scotus } from "./scrapers/scotus.js"; +import { texasLegislature } from "./scrapers/texas-legislature.js"; export const scrapers: readonly Scraper[] = [ federalregister, @@ -11,4 +12,5 @@ export const scrapers: readonly Scraper[] = [ scotus, sccCvig, caSosStatements, + texasLegislature, ]; diff --git a/apps/scraper/src/scrapers/fixtures/texas-hb9-history.xml b/apps/scraper/src/scrapers/fixtures/texas-hb9-history.xml new file mode 100644 index 0000000..b92063f --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/texas-hb9-history.xml @@ -0,0 +1,78 @@ + + + Relating to an exemption from ad valorem taxation. + Meyer | Bonnen + Bernal + Bettencourt + + + Taxation--Property-Exemptions (I0793) + Business & Commerce--General (I0050) + + + + + + + + + + + Introduced + https://capitol.texas.gov/tlodocs/89R/billtext/html/HB00009I.htm + https://capitol.texas.gov/tlodocs/89R/billtext/pdf/HB00009I.pdf + ftp://ftp.legis.state.tx.us/bills/89R/billtext/HTML/house_bills/HB00001_HB00099/HB00009I.HTM + ftp://ftp.legis.state.tx.us/bills/89R/billtext/PDF/house_bills/HB00001_HB00099/HB00009I.PDF + + + Enrolled + https://capitol.texas.gov/tlodocs/89R/billtext/html/HB00009F.htm + https://capitol.texas.gov/tlodocs/89R/billtext/pdf/HB00009F.pdf + ftp://ftp.legis.state.tx.us/bills/89R/billtext/HTML/house_bills/HB00001_HB00099/HB00009F.HTM + ftp://ftp.legis.state.tx.us/bills/89R/billtext/PDF/house_bills/HB00001_HB00099/HB00009F.PDF + + + + + + + House Committee Report + https://capitol.texas.gov/tlodocs/89R/analysis/html/HB00009H.htm + https://capitol.texas.gov/tlodocs/89R/analysis/pdf/HB00009H.pdf + ftp://ftp.legis.state.tx.us/bills/89R/analysis/HTML/house_bills/HB00001_HB00099/HB00009H.HTM + ftp://ftp.legis.state.tx.us/bills/89R/analysis/PDF/house_bills/HB00001_HB00099/HB00009H.PDF + + + + + + + Introduced + https://capitol.texas.gov/tlodocs/89R/fiscalnotes/html/HB00009I.htm + https://capitol.texas.gov/tlodocs/89R/fiscalnotes/pdf/HB00009I.pdf + ftp://ftp.legis.state.tx.us/bills/89R/fiscalNotes/HTML/house_bills/HB00001_HB00099/HB00009I.HTM + ftp://ftp.legis.state.tx.us/bills/89R/fiscalNotes/PDF/house_bills/HB00001_HB00099/HB00009I.PDF + + + + + + + + 06/12/2025 + E100 + Effective immediately + + + 11/12/2024 + H001 + Filed + + + 05/19/2025 + H630 + Record vote + RV#2999 + + + diff --git a/apps/scraper/src/scrapers/texas-legislature-parser.ts b/apps/scraper/src/scrapers/texas-legislature-parser.ts new file mode 100644 index 0000000..7ce062d --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature-parser.ts @@ -0,0 +1,326 @@ +import { load } from "cheerio"; + +export const TEXAS_JURISDICTION = + "ocd-jurisdiction/country:us/state:tx/government"; + +export interface TexasDocument { + type: "bill_text" | "analysis" | "fiscal_note"; + description: string; + htmlUrl?: string; + pdfUrl?: string; + ftpHtmlUrl?: string; + ftpPdfUrl?: string; + text?: string; +} + +export interface TexasVote { + identifier: string; + date?: string; + chamber?: "House" | "Senate"; + motion?: string; + result?: string; + sourceUrl?: string; + counts: { option: string; value: number }[]; + votes: { option: string; voterName: string; openStatesId?: string }[]; +} + +export interface ParsedTexasBill { + billNumber: string; + title: string; + sponsor?: string; + status?: string; + introducedDate?: Date; + chamber: "House" | "Senate"; + jurisdiction: typeof TEXAS_JURISDICTION; + legislativeSession: string; + subjects: string[]; + sponsorships: { + name: string; + classification: "primary" | "cosponsor"; + chamber: "House" | "Senate"; + }[]; + documents: TexasDocument[]; + votes: TexasVote[]; + actions: { date: string; text: string; type?: string }[]; + url: string; +} + +function normalizeSpace(value: string | undefined): string | undefined { + const normalized = value?.replace(/\s+/g, " ").trim(); + return normalized || undefined; +} + +function parseTexasDate(value: string | undefined): string | undefined { + const text = normalizeSpace(value); + if (!text) return undefined; + const match = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/.exec(text); + if (match) { + const [, month, day, year] = match; + return `${year}-${month!.padStart(2, "0")}-${day!.padStart(2, "0")}`; + } + const parsed = new Date(text); + return Number.isNaN(parsed.valueOf()) + ? undefined + : parsed.toISOString().slice(0, 10); +} + +export function normalizeTexasBillNumber(value: string): string { + const match = /(HCR|HJR|HR|HB|SCR|SJR|SR|SB)\s*0*(\d+)/i.exec(value); + if (!match) throw new Error(`Unrecognized Texas bill identifier: ${value}`); + return `${match[1]!.toUpperCase()} ${Number(match[2])}`; +} + +function chamberFor(identifier: string): "House" | "Senate" { + return identifier.startsWith("H") ? "House" : "Senate"; +} + +function splitNames(value: string | undefined): string[] { + return (value ?? "") + .split("|") + .map((name) => normalizeSpace(name)) + .filter((name): name is string => Boolean(name)); +} + +function firstText( + $: ReturnType, + selectors: readonly string[], +): string | undefined { + for (const selector of selectors) { + const value = normalizeSpace($(selector).first().text()); + if (value) return value; + } + return undefined; +} + +function parseDocuments($: ReturnType): TexasDocument[] { + const documents: TexasDocument[] = []; + const groups = [ + ["billtext > docTypes > bill > versions > version", "bill_text"], + ["billtext > docTypes > analysis > versions > version", "analysis"], + ["billtext > docTypes > fiscalNote > versions > version", "fiscal_note"], + ] as const; + + for (const [selector, type] of groups) { + $(selector).each((_, element) => { + const version = $(element); + const description = + normalizeSpace(version.find("versionDescription").first().text()) ?? + type.replace("_", " "); + const htmlUrl = normalizeSpace(version.find("WebHTMLURL").first().text()); + const pdfUrl = normalizeSpace(version.find("WebPDFURL").first().text()); + const ftpHtmlUrl = normalizeSpace( + version.find("FTPHTMLURL").first().text(), + ); + const ftpPdfUrl = normalizeSpace(version.find("FTPPDFURL").first().text()); + if (htmlUrl || pdfUrl || ftpHtmlUrl || ftpPdfUrl) { + documents.push({ + type, + description, + ...(htmlUrl && { htmlUrl }), + ...(pdfUrl && { pdfUrl }), + ...(ftpHtmlUrl && { ftpHtmlUrl }), + ...(ftpPdfUrl && { ftpPdfUrl }), + }); + } + }); + } + return documents; +} + +function parseVotes($: ReturnType): TexasVote[] { + const votes: TexasVote[] = []; + $("votes > vote, recordVotes > vote, voteHistory > vote").each( + (_, element) => { + const vote = $(element); + const text = (selectors: string[]) => { + for (const selector of selectors) { + const value = normalizeSpace(vote.find(selector).first().text()); + if (value) return value; + } + return undefined; + }; + const identifier = text([ + "identifier", + "voteNumber", + "rollCallId", + "actionNumber", + ]); + if (!identifier) return; + const chamberText = text(["chamber", "organization"]); + const chamber = chamberText + ? chamberText.toLowerCase().startsWith("h") + ? "House" + : chamberText.toLowerCase().startsWith("s") + ? "Senate" + : undefined + : undefined; + const counts: TexasVote["counts"] = []; + vote.find("counts > count, totals > total").each((__, countElement) => { + const count = $(countElement); + const option = normalizeSpace( + count.attr("option") ?? count.find("option").first().text(), + ); + const value = Number( + normalizeSpace(count.attr("value") ?? count.find("value").first().text()), + ); + if (option && Number.isInteger(value)) counts.push({ option, value }); + }); + const memberVotes: TexasVote["votes"] = []; + vote.find("voters > voter, memberVotes > vote").each((__, voterElement) => { + const voter = $(voterElement); + const voterName = normalizeSpace( + voter.attr("name") ?? + voter.find("name, voterName, memberName").first().text(), + ); + const option = normalizeSpace( + voter.attr("option") ?? voter.find("option, vote").first().text(), + ); + if (voterName && option) memberVotes.push({ voterName, option }); + }); + votes.push({ + identifier, + ...(parseTexasDate(text(["date", "voteDate"])) && { + date: parseTexasDate(text(["date", "voteDate"])), + }), + ...(chamber && { chamber }), + ...(text(["motion", "motionText", "description"]) && { + motion: text(["motion", "motionText", "description"]), + }), + ...(text(["result", "outcome"]) && { + result: text(["result", "outcome"]), + }), + ...(text(["sourceUrl", "url"]) && { + sourceUrl: text(["sourceUrl", "url"]), + }), + counts, + votes: memberVotes, + }); + }, + ); + $("actions > action").each((_, element) => { + const action = $(element); + if (normalizeSpace(action.find("description").first().text()) !== "Record vote") { + return; + } + const identifier = normalizeSpace(action.find("comment").first().text()); + if (!identifier) return; + const actionNumber = normalizeSpace(action.find("actionNumber").first().text()); + votes.push({ + identifier, + ...(parseTexasDate(action.find("date").first().text()) && { + date: parseTexasDate(action.find("date").first().text()), + }), + ...(actionNumber?.startsWith("H") && { chamber: "House" }), + ...(actionNumber?.startsWith("S") && { chamber: "Senate" }), + motion: "Record vote", + counts: [], + votes: [], + }); + }); + $("committees > house, committees > senate").each((_, element) => { + const committee = $(element); + const name = normalizeSpace(committee.attr("name")); + if (!name) return; + const chamber = element.tagName.toLowerCase() === "house" ? "House" : "Senate"; + const countAttributes = [ + ["Yea", "ayeVotes"], + ["Nay", "nayVotes"], + ["Present not voting", "presentNotVotingVotes"], + ["Absent", "absentVotes"], + ] as const; + const counts = countAttributes.flatMap(([option, attribute]) => { + const value = Number(committee.attr(attribute)); + return Number.isInteger(value) ? [{ option, value }] : []; + }); + if (counts.length === 0) return; + votes.push({ + identifier: `committee:${chamber.toLowerCase()}:${name}`, + chamber, + motion: `${name} committee vote`, + result: normalizeSpace(committee.attr("status")), + counts, + votes: [], + }); + }); + return votes; +} + +export function parseTexasBillHistory( + xml: string, + session: string, +): ParsedTexasBill { + const $ = load(xml, { xml: true }); + const root = $.root().children().first(); + const rawIdentifier = root.attr("bill") ?? firstText($, ["billNumber"]); + if (!rawIdentifier) throw new Error("Texas bill history is missing bill identity"); + const billNumber = normalizeTexasBillNumber(rawIdentifier); + const title = firstText($, ["caption"]); + if (!title || title.includes("Bill does not exist")) { + throw new Error(`${billNumber} does not contain a usable caption`); + } + const chamber = chamberFor(billNumber); + const otherChamber = chamber === "House" ? "Senate" : "House"; + const sponsorships: ParsedTexasBill["sponsorships"] = []; + for (const [selector, classification, sponsorChamber] of [ + ["authors", "primary", chamber], + ["coauthors", "cosponsor", chamber], + ["sponsors", "primary", otherChamber], + ["cosponsors", "cosponsor", otherChamber], + ] as const) { + for (const name of splitNames($(selector).first().text())) { + sponsorships.push({ name, classification, chamber: sponsorChamber }); + } + } + + const actions: ParsedTexasBill["actions"] = []; + $("actions > action").each((_, element) => { + const action = $(element); + const date = parseTexasDate(action.find("date").first().text()); + const text = normalizeSpace(action.find("description").first().text()); + if (!date || !text) return; + const actionNumber = normalizeSpace(action.find("actionNumber").first().text()); + actions.push({ date, text, ...(actionNumber && { type: actionNumber }) }); + }); + actions.sort((left, right) => left.date.localeCompare(right.date)); + + const sponsor = sponsorships + .filter((item) => item.classification === "primary") + .map((item) => item.name) + .join(" | ") + .slice(0, 256); + const billSlug = billNumber.replace(/\s+/g, ""); + return { + billNumber, + title, + ...(sponsor && { sponsor }), + ...(actions.at(-1)?.text && { status: actions.at(-1)!.text.slice(0, 100) }), + ...(actions[0]?.date && { + introducedDate: new Date(`${actions[0].date}T12:00:00.000Z`), + }), + chamber, + jurisdiction: TEXAS_JURISDICTION, + legislativeSession: session.toUpperCase(), + subjects: $("subjects > subject") + .map((_, element) => normalizeSpace($(element).text())) + .get() + .filter((value): value is string => Boolean(value)), + sponsorships, + documents: parseDocuments($), + votes: parseVotes($), + actions, + url: `https://capitol.texas.gov/BillLookup/History.aspx?LegSess=${encodeURIComponent(session.toUpperCase())}&Bill=${billSlug}`, + }; +} + +export function htmlToText(html: string): string { + const $ = load(html); + $("script, style, nav, header, footer").remove(); + return $.root().text().replace(/\s+/g, " ").trim(); +} + +export function openStatesSessionName(session: string): string { + const normalized = session.toUpperCase(); + if (/^\d{2}R$/.test(normalized)) return normalized.slice(0, 2); + if (/^\d{3}$/.test(normalized)) return normalized; + throw new Error(`Invalid Texas legislative session: ${session}`); +} diff --git a/apps/scraper/src/scrapers/texas-legislature-source.ts b/apps/scraper/src/scrapers/texas-legislature-source.ts new file mode 100644 index 0000000..4eba3d3 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature-source.ts @@ -0,0 +1,134 @@ +import { Writable } from "node:stream"; + +import { Client } from "basic-ftp"; + +import type { TexasDocument } from "./texas-legislature-parser.js"; + +export interface BulkEntry { + name: string; + isDirectory: boolean; +} + +export interface TexasBulkClient { + list(path: string): Promise; + download(path: string): Promise; + close(): void; +} + +export class TexasFtpClient implements TexasBulkClient { + private readonly client = new Client(30_000); + + async connect(): Promise { + await this.client.access({ + host: "ftp.legis.state.tx.us", + user: "anonymous", + password: "billion@example.invalid", + secure: false, + }); + } + + async list(path: string): Promise { + return (await this.client.list(path)).map((entry) => ({ + name: entry.name, + isDirectory: entry.isDirectory, + })); + } + + async download(path: string): Promise { + const chunks: Buffer[] = []; + const sink = new Writable({ + write(chunk: Buffer | string, _encoding, callback) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + callback(); + }, + }); + await this.client.downloadTo(sink, path); + return Buffer.concat(chunks); + } + + close(): void { + this.client.close(); + } +} + +export function selectCurrentTexasSession(names: readonly string[]): string { + const sessions = names + .map((name) => name.toUpperCase()) + .filter((name) => /^\d{2}(?:R|\d)$/.test(name)) + .sort((left, right) => { + const legislature = Number(left.slice(0, 2)) - Number(right.slice(0, 2)); + if (legislature !== 0) return legislature; + const rank = (value: string) => + value[2] === "R" ? 0 : Number(value[2]) || 0; + return rank(left) - rank(right); + }); + const session = sessions.at(-1); + if (!session) throw new Error("No Texas legislative sessions found in /bills"); + return session; +} + +export async function listFilesRecursively( + client: TexasBulkClient, + root: string, +): Promise { + const entries = (await client.list(root)).sort((a, b) => + a.name.localeCompare(b.name), + ); + const files: string[] = []; + for (const entry of entries) { + if (entry.name === "." || entry.name === "..") continue; + const child = `${root.replace(/\/$/, "")}/${entry.name}`; + if (entry.isDirectory) files.push(...(await listFilesRecursively(client, child))); + else files.push(child); + } + return files; +} + +const folderByPrefix: Record = { + HB: "house_bills", + HCR: "house_concurrent_resolutions", + HJR: "house_joint_resolutions", + HR: "house_resolutions", + SB: "senate_bills", + SCR: "senate_concurrent_resolutions", + SJR: "senate_joint_resolutions", + SR: "senate_resolutions", +}; + +const rootByType: Record = { + bill_text: "billtext", + analysis: "analysis", + fiscal_note: "fiscalnotes", +}; + +export function bulkHtmlPath( + session: string, + document: Pick, +): string | undefined { + if (document.ftpHtmlUrl) { + try { + return decodeURIComponent(new URL(document.ftpHtmlUrl).pathname); + } catch { + return undefined; + } + } + if (!document.htmlUrl) return undefined; + let filename: string; + try { + filename = new URL(document.htmlUrl).pathname.split("/").at(-1) ?? ""; + } catch { + return undefined; + } + const match = /^(HCR|HJR|HR|HB|SCR|SJR|SR|SB)(\d{5})[A-Z0-9]*\.html?$/i.exec( + filename, + ); + if (!match) return undefined; + const prefix = match[1]!.toUpperCase(); + const number = Number(match[2]); + const start = number < 100 ? 1 : Math.floor(number / 100) * 100; + const end = number < 100 ? 99 : start + 99; + const pad = (value: number) => String(value).padStart(5, "0"); + const documentRoot = + document.type === "fiscal_note" ? "fiscalNotes" : rootByType[document.type]; + return `/bills/${session.toUpperCase()}/${documentRoot}/HTML/${folderByPrefix[prefix]}/${prefix}${pad(start)}_${prefix}${pad(end)}/${filename}`; +} diff --git a/apps/scraper/src/scrapers/texas-legislature.config.ts b/apps/scraper/src/scrapers/texas-legislature.config.ts new file mode 100644 index 0000000..8e4be62 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature.config.ts @@ -0,0 +1,16 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const texasLegislatureConfig = { + id: "texas-legislature", + name: "Texas Legislature Online", + source: + "Texas Legislative Council anonymous FTP — current-session history XML and bulk documents", + environment: { + required: ["POSTGRES_URL"], + optional: [ + "OPEN_STATES_API_KEY", + "TEXAS_LEGISLATURE_SESSION", + "TEXAS_LEGISLATURE_MAX_ITEMS", + ], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/texas-legislature.test.ts b/apps/scraper/src/scrapers/texas-legislature.test.ts new file mode 100644 index 0000000..e51d886 --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature.test.ts @@ -0,0 +1,105 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { CreateBillSchema } from "@acme/db/schema"; + +import { + htmlToText, + openStatesSessionName, + parseTexasBillHistory, +} from "./texas-legislature-parser.js"; +import { + bulkHtmlPath, + listFilesRecursively, + selectCurrentTexasSession, + type TexasBulkClient, +} from "./texas-legislature-source.js"; + +const fixture = await readFile( + new URL("./fixtures/texas-hb9-history.xml", import.meta.url), + "utf8", +); + +void test("parses official-style Texas bill history deterministically", () => { + const bill = parseTexasBillHistory(fixture, "89R"); + assert.equal(bill.billNumber, "HB 9"); + assert.equal(bill.chamber, "House"); + assert.equal(bill.legislativeSession, "89R"); + assert.equal(bill.sponsor, "Meyer | Bonnen | Bettencourt"); + assert.equal(bill.status, "Effective immediately"); + assert.equal(bill.introducedDate?.toISOString(), "2024-11-12T12:00:00.000Z"); + assert.deepEqual( + bill.documents.map((document) => document.type), + ["bill_text", "bill_text", "analysis", "fiscal_note"], + ); + assert.deepEqual(bill.votes[0], { + identifier: "RV#2999", + date: "2025-05-19", + chamber: "House", + motion: "Record vote", + counts: [], + votes: [], + }); + assert.deepEqual(bill.votes[1]?.counts, [ + { option: "Yea", value: 9 }, + { option: "Nay", value: 1 }, + { option: "Present not voting", value: 0 }, + { option: "Absent", value: 1 }, + ]); +}); + +void test("maps public document names to the official FTP bulk tree", () => { + const bill = parseTexasBillHistory(fixture, "89R"); + assert.equal( + bulkHtmlPath("89R", bill.documents[0]!), + "/bills/89R/billtext/HTML/house_bills/HB00001_HB00099/HB00009I.HTM", + ); + assert.equal( + bulkHtmlPath("89R", bill.documents.at(-1)!), + "/bills/89R/fiscalNotes/HTML/house_bills/HB00001_HB00099/HB00009I.HTM", + ); +}); + +void test("selects only the latest Texas session and maps Open States names", () => { + assert.equal(selectCurrentTexasSession(["88R", "89R", "891", "892"]), "892"); + assert.equal(openStatesSessionName("89R"), "89"); + assert.equal(openStatesSessionName("892"), "892"); +}); + +void test("walks bulk directories in stable lexical order", async () => { + const listings: Record = { + "/root": [ + { name: "z.xml", isDirectory: false }, + { name: "a", isDirectory: true }, + ], + "/root/a": [{ name: "b.xml", isDirectory: false }], + }; + const client: TexasBulkClient = { + list: async (path) => listings[path] ?? [], + download: async () => Buffer.alloc(0), + close: () => undefined, + }; + assert.deepEqual(await listFilesRecursively(client, "/root"), [ + "/root/a/b.xml", + "/root/z.xml", + ]); +}); + +void test("extracts readable deterministic text from bulk HTML", () => { + assert.equal( + htmlToText("

Bill text

Section 1.

"), + "Bill text Section 1.", + ); +}); + +void test("parsed Texas data satisfies the shared persisted bill contract", () => { + const parsed = parseTexasBillHistory(fixture, "89R"); + assert.equal( + CreateBillSchema.safeParse({ + ...parsed, + sourceWebsite: "capitol.texas.gov", + }).success, + true, + ); +}); diff --git a/apps/scraper/src/scrapers/texas-legislature.ts b/apps/scraper/src/scrapers/texas-legislature.ts new file mode 100644 index 0000000..000b63d --- /dev/null +++ b/apps/scraper/src/scrapers/texas-legislature.ts @@ -0,0 +1,162 @@ +import type { BillData, Scraper } from "../utils/types.js"; +import { upsertContent } from "../utils/db/operations.js"; +import { createLogger } from "../utils/log.js"; +import { setExpectedTotal } from "../utils/progress.js"; +import { + htmlToText, + openStatesSessionName, + parseTexasBillHistory, + TEXAS_JURISDICTION, +} from "./texas-legislature-parser.js"; +import { + bulkHtmlPath, + listFilesRecursively, + selectCurrentTexasSession, + TexasFtpClient, + type TexasBulkClient, +} from "./texas-legislature-source.js"; +import { texasLegislatureConfig } from "./texas-legislature.config.js"; + +const logger = createLogger("texas-legislature"); + +interface OpenStatesSearchResponse { + results?: { id: string; identifier: string; session: string }[]; +} + +export async function matchOpenStatesBillId( + session: string, + billNumber: string, + apiKey = process.env.OPEN_STATES_API_KEY, +): Promise { + if (!apiKey) return undefined; + const url = new URL("https://v3.openstates.org/bills"); + url.searchParams.set("jurisdiction", TEXAS_JURISDICTION); + url.searchParams.set("session", openStatesSessionName(session)); + url.searchParams.set("q", billNumber); + url.searchParams.set("per_page", "5"); + const response = await fetch(url, { + headers: { Accept: "application/json", "X-API-KEY": apiKey }, + }); + if (!response.ok) return undefined; + const payload = (await response.json()) as OpenStatesSearchResponse; + return payload.results?.find( + (bill) => + bill.identifier.replace(/\s+/g, "").toUpperCase() === + billNumber.replace(/\s+/g, "").toUpperCase() && + bill.session === openStatesSessionName(session), + )?.id; +} + +async function enrichDocuments( + client: TexasBulkClient, + session: string, + documents: ReturnType["documents"], +) { + return Promise.all( + documents.map(async (document) => { + const path = bulkHtmlPath(session, document); + if (!path) return document; + try { + const html = (await client.download(path)).toString("utf8"); + const text = htmlToText(html); + return text ? { ...document, text } : document; + } catch (error) { + logger.debug( + `Optional bulk HTML unavailable at ${path}: ${error instanceof Error ? error.message : error}`, + ); + return document; + } + }), + ); +} + +export async function scrapeTexasLegislature( + client: TexasBulkClient, + options: { maxItems: number; session?: string }, +): Promise { + const availableSessions = (await client.list("/bills")) + .filter((entry) => entry.isDirectory) + .map((entry) => entry.name); + const currentSession = selectCurrentTexasSession(availableSessions); + const session = options.session?.toUpperCase() ?? currentSession; + if (session !== currentSession) { + throw new Error( + `Texas ingestion is current-session only: requested ${session}, latest bulk session is ${currentSession}`, + ); + } + + logger.info(`Reading official Texas bulk data for ${session}...`); + const paths = (await listFilesRecursively( + client, + `/bills/${session}/billhistory`, + )) + .filter( + (path) => + path.toLowerCase().endsWith(".xml") && + !/\/history(?:_periodic)?\.xml$/i.test(path), + ) + .sort() + .slice(0, options.maxItems); + setExpectedTotal(paths.length); + let persisted = 0; + + for (const path of paths) { + try { + const parsed = parseTexasBillHistory( + (await client.download(path)).toString("utf8"), + session, + ); + const documents = await enrichDocuments(client, session, parsed.documents); + const latestBillText = documents + .filter((document) => document.type === "bill_text" && document.text) + .at(-1)?.text; + let openStatesId: string | undefined; + try { + openStatesId = await matchOpenStatesBillId(session, parsed.billNumber); + } catch (error) { + logger.debug( + `Open States identity match skipped for ${parsed.billNumber}: ${error instanceof Error ? error.message : error}`, + ); + } + const data: BillData = { + ...parsed, + documents, + ...(latestBillText && { fullText: latestBillText }), + ...(openStatesId && { openStatesId }), + sourceWebsite: "capitol.texas.gov", + }; + await upsertContent( + { type: "bill", data }, + { skipEnrichment: true }, + ); + persisted += 1; + } catch (error) { + logger.error( + `Failed to process ${path}: ${error instanceof Error ? error.message : error}`, + ); + } + } + logger.success(`Persisted ${persisted}/${paths.length} Texas bills from ${session}.`); +} + +async function scrape(options?: { maxItems?: number }): Promise { + const client = new TexasFtpClient(); + try { + await client.connect(); + await scrapeTexasLegislature(client, { + maxItems: + options?.maxItems ?? + (Number(process.env.TEXAS_LEGISLATURE_MAX_ITEMS) || 100), + ...(process.env.TEXAS_LEGISLATURE_SESSION && { + session: process.env.TEXAS_LEGISLATURE_SESSION, + }), + }); + } finally { + client.close(); + } +} + +export const texasLegislature: Scraper = { + ...texasLegislatureConfig, + scrape, +}; diff --git a/apps/scraper/src/utils/db/helpers.ts b/apps/scraper/src/utils/db/helpers.ts index 0d0ce7a..2e1aaa8 100644 --- a/apps/scraper/src/utils/db/helpers.ts +++ b/apps/scraper/src/utils/db/helpers.ts @@ -20,6 +20,7 @@ const logger = createLogger("db"); export async function checkExistingBill( billNumber: string, sourceWebsite: string, + legislativeSession = "", ): Promise { try { const [existing] = await db @@ -30,7 +31,13 @@ export async function checkExistingBill( thumbnailUrl: Bill.thumbnailUrl, }) .from(Bill) - .where(and(eq(Bill.billNumber, billNumber), eq(Bill.sourceWebsite, sourceWebsite))) + .where( + and( + eq(Bill.billNumber, billNumber), + eq(Bill.sourceWebsite, sourceWebsite), + eq(Bill.legislativeSession, legislativeSession), + ), + ) .limit(1); if (!existing) { diff --git a/apps/scraper/src/utils/db/operations.ts b/apps/scraper/src/utils/db/operations.ts index 1e05213..85cf1a1 100644 --- a/apps/scraper/src/utils/db/operations.ts +++ b/apps/scraper/src/utils/db/operations.ts @@ -70,6 +70,14 @@ function hashFields(input: ContentData): string { status: input.data.status, summary: input.data.summary, fullText: input.data.fullText, + jurisdiction: input.data.jurisdiction, + legislativeSession: input.data.legislativeSession, + openStatesId: input.data.openStatesId, + subjects: input.data.subjects, + sponsorships: input.data.sponsorships, + documents: input.data.documents, + votes: input.data.votes, + actions: input.data.actions, }); case "government_content": return JSON.stringify({ @@ -90,7 +98,11 @@ function hashFields(input: ContentData): string { async function checkExisting(input: ContentData) { switch (input.type) { case "bill": - return checkExistingBill(input.data.billNumber, input.data.sourceWebsite); + return checkExistingBill( + input.data.billNumber, + input.data.sourceWebsite, + input.data.legislativeSession, + ); case "government_content": return checkExistingGovernmentContent(input.data.url); case "court_case": @@ -111,7 +123,7 @@ function getUpdateTable(input: ContentData) { export async function upsertContent( input: ContentData, - options?: { newItemLimiter?: NewItemLimiter }, + options?: { newItemLimiter?: NewItemLimiter; skipEnrichment?: boolean }, ) { const newContentHash = createContentHash(hashFields(input)); const existing = await checkExisting(input); @@ -210,7 +222,11 @@ export async function upsertContent( versions: [], }) .onConflictDoUpdate({ - target: [Bill.billNumber, Bill.sourceWebsite], + target: [ + Bill.billNumber, + Bill.sourceWebsite, + Bill.legislativeSession, + ], set: { title: d.title, description: d.description, @@ -219,8 +235,16 @@ export async function upsertContent( introducedDate: d.introducedDate, congress: d.congress, chamber: d.chamber, + jurisdiction: d.jurisdiction, + legislativeSession: d.legislativeSession, + openStatesId: d.openStatesId, + subjects: d.subjects, + sponsorships: d.sponsorships, + documents: d.documents, + votes: d.votes, summary: d.summary, fullText: d.fullText, + actions: d.actions, url: d.url, contentHash: newContentHash, updatedAt: new Date(), @@ -290,6 +314,15 @@ export async function upsertContent( return result; } + if (options?.skipEnrichment) { + tickProgress({ + newEntries: progressKind === "new" ? 1 : 0, + unchanged: progressKind === "unchanged" ? 1 : 0, + changed: progressKind === "changed" ? 1 : 0, + }); + return result; + } + // Phase 2: AI enrichment — skipped entirely if rate-limited try { const existingDescription = sourceDescription || persistedDescription; diff --git a/docs/scraper.md b/docs/scraper.md index 68a83d3..af53f98 100644 --- a/docs/scraper.md +++ b/docs/scraper.md @@ -25,6 +25,7 @@ process environment at runtime, not embedded during the build. | `ca-sos-statements.ts` | CA Secretary of State guide | `civic_api_cache` | official candidate-statement pages | | `ca-lao-fiscal.ts` | CA LAO ballot analyses | `civic_api_cache` | proposition fiscal analyses via HTML parse | | `ca-vig-archive.ts` | CA SOS voter-guide archive | `civic_api_cache` | historical proposition guide pages via HTML parse | +| `texas-legislature.ts` | Texas Legislative Council FTP | `bill` | current-session XML + bulk HTML; no site mining | All HTTP goes through one `fetchWithRetry()` utility (`apps/scraper/src/utils/fetch.ts`): exponential backoff (1s/2s/4s…), `Retry-After` support (seconds or HTTP-date), 30s default timeout via `AbortController`, retriable on 429/5xx and `ECONNRESET`/`ECONNREFUSED`, plus a stateful **per-host backoff** that ramps on 429/5xx and relaxes on success. @@ -35,12 +36,18 @@ All HTTP goes through one `fetchWithRetry()` utility (`apps/scraper/src/utils/fe `apps/scraper/src/utils/db/operations.ts` centralizes writes behind a discriminated-union `upsertContent(type, data)` (`type` ∈ bill | government_content | court_case). Each run: 1. Compute a SHA-256 over the type-specific key fields (title, summary, full text, status…). -2. Look up the existing row by its natural key (`(billNumber, sourceWebsite)`, `url`, or `caseNumber`). +2. Look up the existing row by its natural key (`(billNumber, sourceWebsite, legislativeSession)`, `url`, or `caseNumber`). 3. **Unchanged hash** → skip AI entirely; backfill only missing AI assets. 4. **New or changed** → run the AI pipeline, upsert via `onConflictDoUpdate`, append to `versions`. `SCRAPER_FORCE_AI_REGEN=1` overrides the cache. A `isUsableText()` gate refuses to feed AI any text under 200 chars or that's mostly blank/all-caps/single-word lines — keeps the model from "summarizing" garbage. +Texas is the provider-neutral state-bill extension to this flow. Its rows carry +an OCD jurisdiction, legislative session, subjects, sponsorships, documents, +votes, and an optional exact Open States ID. The scraper selects only the latest +FTP session and sets `skipEnrichment`; the API exposes only that newest session +through `content.texasBills`, with full supporting material in `content.getById`. + ## AI Pipeline Provider config lives in `apps/scraper/src/utils/ai/provider.ts`: text via **OpenRouter** (Vercel AI SDK) using `OPENROUTER_MODEL`, which defaults to **`deepseek/deepseek-v4-flash`**; deprecated direct DeepSeek credentials remain a migration fallback. PDF vision fallback uses **Gemini `gemini-2.5-flash`**, and images use **Black Forest Labs FLUX.2 Klein 9B**. Provider usage and image costs are tracked per run. diff --git a/packages/api/src/router/content.ts b/packages/api/src/router/content.ts index ffd9c81..e539fe2 100644 --- a/packages/api/src/router/content.ts +++ b/packages/api/src/router/content.ts @@ -184,6 +184,60 @@ const _ContentDetailSchema = ContentCardSchema.extend({ export type ContentDetail = z.infer; export const contentRouter = { + // Read the latest Texas bulk session persisted by the scraper. This route is + // intentionally current-session only; it is not a historical session browser. + texasBills: publicProcedure + .input( + z + .object({ + limit: z.number().int().min(1).max(50).default(20), + cursor: z.number().int().min(0).default(0), + }) + .optional(), + ) + .query(async ({ input }) => { + const jurisdiction = + "ocd-jurisdiction/country:us/state:tx/government"; + const [latest] = await db + .select({ legislativeSession: Bill.legislativeSession }) + .from(Bill) + .where(eq(Bill.jurisdiction, jurisdiction)) + .orderBy(desc(Bill.updatedAt), desc(Bill.createdAt)) + .limit(1); + if (!latest) return { items: [], nextCursor: undefined }; + const limit = input?.limit ?? 20; + const cursor = input?.cursor ?? 0; + const rows = await db + .select({ + id: Bill.id, + billNumber: Bill.billNumber, + title: Bill.title, + description: Bill.description, + summary: Bill.summary, + status: Bill.status, + chamber: Bill.chamber, + legislativeSession: Bill.legislativeSession, + openStatesId: Bill.openStatesId, + subjects: Bill.subjects, + updatedAt: Bill.updatedAt, + }) + .from(Bill) + .where( + and( + eq(Bill.jurisdiction, jurisdiction), + eq(Bill.legislativeSession, latest.legislativeSession), + ), + ) + .orderBy(desc(Bill.updatedAt), desc(Bill.billNumber)) + .limit(limit + 1) + .offset(cursor); + const hasMore = rows.length > limit; + return { + items: hasMore ? rows.slice(0, limit) : rows, + nextCursor: hasMore ? cursor + limit : undefined, + }; + }), + // Get all content from database getAll: publicProcedure.query(async () => { const bills = await db @@ -579,6 +633,13 @@ export const contentRouter = { type?: string; }[], status: b.status ?? undefined, + jurisdiction: b.jurisdiction, + legislativeSession: b.legislativeSession || undefined, + openStatesId: b.openStatesId ?? undefined, + subjects: b.subjects ?? [], + sponsorships: b.sponsorships ?? [], + documents: b.documents ?? [], + votes: b.votes ?? [], lensData: await getLensData(b.id, "bill"), }, ]); diff --git a/packages/db/migrations/add_state_legislation_fields.sql b/packages/db/migrations/add_state_legislation_fields.sql new file mode 100644 index 0000000..7d530fa --- /dev/null +++ b/packages/db/migrations/add_state_legislation_fields.sql @@ -0,0 +1,14 @@ +ALTER TABLE "bill" + ADD COLUMN IF NOT EXISTS "jurisdiction" varchar(100) DEFAULT 'ocd-jurisdiction/country:us/government' NOT NULL, + ADD COLUMN IF NOT EXISTS "legislative_session" varchar(20) DEFAULT '' NOT NULL, + ADD COLUMN IF NOT EXISTS "open_states_id" text, + ADD COLUMN IF NOT EXISTS "subjects" jsonb DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS "sponsorships" jsonb DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS "documents" jsonb DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS "votes" jsonb DEFAULT '[]'::jsonb; + +ALTER TABLE "bill" DROP CONSTRAINT IF EXISTS "bill_bill_number_source_website_unique"; +ALTER TABLE "bill" DROP CONSTRAINT IF EXISTS "bill_bill_number_source_website_legislative_session_unique"; +ALTER TABLE "bill" + ADD CONSTRAINT "bill_bill_number_source_website_legislative_session_unique" + UNIQUE("bill_number", "source_website", "legislative_session"); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 3c990ba..c653c45 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -56,6 +56,52 @@ export const Bill = pgTable( introducedDate: t.timestamp(), congress: t.integer(), // e.g., 118 for 118th Congress chamber: t.varchar({ length: 50 }), // "House" or "Senate" + jurisdiction: t + .varchar({ length: 100 }) + .notNull() + .default("ocd-jurisdiction/country:us/government"), + legislativeSession: t.varchar({ length: 20 }).notNull().default(""), + openStatesId: t.text(), + subjects: t.jsonb().$type().default([]), + sponsorships: t + .jsonb() + .$type< + { + name: string; + classification: "primary" | "cosponsor"; + chamber: "House" | "Senate"; + }[] + >() + .default([]), + documents: t + .jsonb() + .$type< + { + type: "bill_text" | "analysis" | "fiscal_note"; + description: string; + htmlUrl?: string; + pdfUrl?: string; + ftpHtmlUrl?: string; + ftpPdfUrl?: string; + text?: string; + }[] + >() + .default([]), + votes: t + .jsonb() + .$type< + { + identifier: string; + date?: string; + chamber?: "House" | "Senate"; + motion?: string; + result?: string; + sourceUrl?: string; + counts: { option: string; value: number }[]; + votes: { option: string; voterName: string; openStatesId?: string }[]; + }[] + >() + .default([]), summary: t.text(), fullText: t.text(), aiGeneratedArticle: t.text(), // AI-generated accessible article version @@ -95,7 +141,11 @@ export const Bill = pgTable( ), }), (table) => ({ - uniqueBillNumberSource: unique().on(table.billNumber, table.sourceWebsite), + uniqueBillNumberSourceSession: unique().on( + table.billNumber, + table.sourceWebsite, + table.legislativeSession, + ), searchVectorIdx: index("bill_search_vector_idx").using( "gin", table.searchVector, diff --git a/packages/env/src/registry.ts b/packages/env/src/registry.ts index 6686f51..b63cdc0 100644 --- a/packages/env/src/registry.ts +++ b/packages/env/src/registry.ts @@ -85,6 +85,11 @@ const scraperSourceLimitDefinitions = [ ["SCOTUS_MAX_ITEMS", "CourtListener opinion clusters per run.", "50"], ["SCC_CVIG_MAX_ITEMS", "Santa Clara voter-guide PDFs per run.", "10"], ["CA_SOS_MAX_ITEMS", "California SOS office pages per run.", "9"], + [ + "TEXAS_LEGISLATURE_MAX_ITEMS", + "Texas Legislature bills per current-session run.", + "100", + ], ] as const; export const envRegistry = [ @@ -346,6 +351,15 @@ export const envRegistry = [ requirements: {}, schema: string, }), + define({ + key: "TEXAS_LEGISLATURE_SESSION", + description: + "Optional current Texas bulk session assertion (for example 892); ingestion rejects historical sessions.", + group: "Scraper sources", + secret: false, + requirements: {}, + schema: string.regex(/^\d{2}(?:R|\d)$/i, "must look like 89R or 892"), + }), define({ key: "BFL_API_KEY", description: "Black Forest Labs key for generated feed-card images.", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 27a13bf..a217542 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -462,6 +462,9 @@ importers: ai: specifier: ^6.0.141 version: 6.0.141(zod@4.3.6) + basic-ftp: + specifier: ^6.0.1 + version: 6.0.1 cheerio: specifier: ^1.2.0 version: 1.2.0 @@ -4659,6 +4662,10 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + basic-ftp@6.0.1: + resolution: {integrity: sha512-3ilxa3n4276wGQp/ImRAuz4ALdsj/2Wd3FqoZBZlajDYnByCZ0JMb4+26Rde0wGXIbM0G2HWSfr/Fi8b21KX8g==} + engines: {node: '>=10.0.0'} + better-auth@1.6.13: resolution: {integrity: sha512-jn8ATnGWDzMwpO4a/3iyW1/RayOF/aoPQOfAeqyCVnQCdqkaONVas9CjbY6PovMsTMa/MG+GRABySfzqtj5J/g==} peerDependencies: @@ -13137,6 +13144,8 @@ snapshots: - '@cloudflare/workers-types' - '@opentelemetry/api' + basic-ftp@6.0.1: {} + better-auth@1.6.13(@opentelemetry/api@1.9.0)(@prisma/client@5.22.0(prisma@5.22.0))(better-sqlite3@12.8.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@neondatabase/serverless@0.10.0)(@opentelemetry/api@1.9.0)(@planetscale/database@1.19.0)(@prisma/client@5.22.0(prisma@5.22.0))(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(@vercel/postgres@0.10.0(utf-8-validate@6.0.4))(better-sqlite3@12.8.0)(gel@2.0.0)(kysely@0.28.17)(mysql2@3.11.3)(pg@8.20.0)(postgres@3.4.4)(prisma@5.22.0))(mysql2@3.11.3)(next@16.2.7(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(pg@8.20.0)(prisma@5.22.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3): dependencies: '@better-auth/core': 1.6.13(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.0)(better-call@1.3.5(zod@4.3.6))(jose@6.2.2)(kysely@0.28.17)(nanostores@1.2.0)