From 3b403c452e59f9a59c0f4bcb4f9281c6474ec538 Mon Sep 17 00:00:00 2001 From: ThatXliner Date: Wed, 22 Jul 2026 09:00:13 -0700 Subject: [PATCH] Add Kansas City Council Legistar scraper --- apps/scraper/README.md | 21 + apps/scraper/src/scraper-contracts.ts | 2 + apps/scraper/src/scrapers.ts | 2 + .../fixtures/kansas-city-council/events.json | 86 +++ .../fixtures/kansas-city-council/items.json | 75 ++ .../fixtures/kansas-city-council/votes.json | 20 + .../scrapers/kansas-city-council.config.ts | 11 + .../src/scrapers/kansas-city-council.test.ts | 112 +++ .../src/scrapers/kansas-city-council.ts | 654 ++++++++++++++++++ docs/data-layer.md | 7 +- docs/data-sources-api.md | 20 +- docs/scraper.md | 23 + packages/env/src/registry.ts | 5 + 13 files changed, 1026 insertions(+), 12 deletions(-) create mode 100644 apps/scraper/src/scrapers/fixtures/kansas-city-council/events.json create mode 100644 apps/scraper/src/scrapers/fixtures/kansas-city-council/items.json create mode 100644 apps/scraper/src/scrapers/fixtures/kansas-city-council/votes.json create mode 100644 apps/scraper/src/scrapers/kansas-city-council.config.ts create mode 100644 apps/scraper/src/scrapers/kansas-city-council.test.ts create mode 100644 apps/scraper/src/scrapers/kansas-city-council.ts diff --git a/apps/scraper/README.md b/apps/scraper/README.md index aca4ed6..9775d46 100644 --- a/apps/scraper/README.md +++ b/apps/scraper/README.md @@ -10,6 +10,7 @@ These data sources are registered and run by `all`: | ------------------------ | ------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- | | `federalregister` | Federal Register API presidential documents, then each document's body HTML | `government_content`; AI article/summary and feed-image enrichment | | `durham-bocc` | Durham County's official Legistar API (current election cycle only) | Provider-neutral meetings, agenda items, actions, votes, and official document links; no AI required | +| `kansas-city-council` | Kansas City's official Legistar API (current Council term and body ID 138 only) | Provider-neutral meetings, revisions, documents, legislation, actions, and named votes | | `congress` | Congress.gov API bill list, detail, CRS summaries, formatted text, and legislative actions | `bill`; powers federal bill content and AI/feed enrichment | | `scotus` | CourtListener opinion clusters, dockets, and sub-opinion text for the Supreme Court | `court_case`; powers court content and AI/feed enrichment | | `scc-cvig` | Hand-configured Santa Clara County voter-guide PDFs | Candidate statements in `CivicApiCache`; the API matches statements to candidates | @@ -139,6 +140,7 @@ CONGRESS_MAX_ITEMS=10 pnpm --filter @acme/scraper run start congress | `TX_SOS_MAX_ITEMS` | 12 | Current-cycle Texas SOS election payloads | | `CEDAR_PARK_COUNCIL_MAX_ITEMS` | 100 | Council meetings (after the 12-month cutoff) | | `DURHAM_BOCC_MAX_ITEMS` | 100 | Current-cycle Durham County BOCC meetings | +| `KANSAS_CITY_COUNCIL_MAX_ITEMS` | 250 | Current-term Kansas City Council meetings | | `SCRAPER_MAX_NEW_ITEMS_PER_RUN` | 10 | New records receiving expensive AI/image enrichment | These are per-run limits, not durable calendar-day quotas. Schedule one run per @@ -153,6 +155,25 @@ history plus candidate contact/address fields. See [`docs/ncsbe-election-data.md`](../../docs/ncsbe-election-data.md) for discovery, idempotency, provenance, API, and deterministic Civic-matching details. +## Kansas City Council (`kansas-city-council.ts`) + +This keyless adapter uses `webapi.legistar.com/v1/kansascity`, restricted to +Council body ID `138` and the active term beginning August 1, 2023. It stores +stable Event/EventItem/Vote IDs, Central Time/DST, cancellations and revisions, +locations, official agenda/minutes/attachment links, video media IDs, +legislation file numbers, actions/outcomes, and named votes. It also checks +actioned items because Kansas City can publish a vote while leaving the +Legistar roll-call flag unset. Hidden test events are excluded. + +Meetings are serialized and vote lookups are capped at two concurrent requests. +Reruns upsert natural keys, mark replaced documents non-current, replace a +successfully refreshed roll call atomically, and preserve last-known votes when +the vote endpoint is temporarily unavailable. + +```bash +pnpm --filter @acme/scraper run start kansas-city-council --max-items 5 +``` + ## Cedar Park City Council (`civicengage.ts`) Cedar Park's public site is CivicEngage, but the City Council records page now diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts index 5e78aa8..81e0f20 100644 --- a/apps/scraper/src/scraper-contracts.ts +++ b/apps/scraper/src/scraper-contracts.ts @@ -6,6 +6,7 @@ import { congressConfig } from "./scrapers/congress.config.js"; import { durhamBoccConfig } from "./scrapers/durham-bocc.config.js"; import { durhamOnBaseConfig } from "./scrapers/durham-onbase.config.js"; import { federalregisterConfig } from "./scrapers/federalregister.config.js"; +import { kansasCityCouncilConfig } from "./scrapers/kansas-city-council.config.js"; import { ncsbeConfig } from "./scrapers/ncsbe.config.js"; import { sccCvigConfig } from "./scrapers/scc-cvig.config.js"; import { scotusConfig } from "./scrapers/scotus.config.js"; @@ -14,6 +15,7 @@ import { texasLegislatureConfig } from "./scrapers/texas-legislature.config.js"; export const scraperContracts: readonly ScraperEnvContract[] = [ federalregisterConfig, + kansasCityCouncilConfig, durhamBoccConfig, congressConfig, scotusConfig, diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts index 2e3ad49..51001d4 100644 --- a/apps/scraper/src/scrapers.ts +++ b/apps/scraper/src/scrapers.ts @@ -5,6 +5,7 @@ import { congress } from "./scrapers/congress.js"; import { durhamBocc } from "./scrapers/durham-bocc.js"; import { durhamOnBase } from "./scrapers/durham-onbase.js"; import { federalregister } from "./scrapers/federalregister.js"; +import { kansasCityCouncil } from "./scrapers/kansas-city-council.js"; import { ncsbe } from "./scrapers/ncsbe.js"; import { sccCvig } from "./scrapers/scc-cvig.js"; import { scotus } from "./scrapers/scotus.js"; @@ -13,6 +14,7 @@ import { texasLegislature } from "./scrapers/texas-legislature.js"; export const scrapers: readonly Scraper[] = [ federalregister, + kansasCityCouncil, durhamBocc, congress, scotus, diff --git a/apps/scraper/src/scrapers/fixtures/kansas-city-council/events.json b/apps/scraper/src/scrapers/fixtures/kansas-city-council/events.json new file mode 100644 index 0000000..cdf7266 --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/kansas-city-council/events.json @@ -0,0 +1,86 @@ +[ + { + "EventId": 19001, + "EventGuid": "11111111-1111-4111-8111-111111111111", + "EventLastModifiedUtc": "2026-01-09T22:00:00.000", + "EventRowVersion": "kc-event-v1", + "EventBodyId": 138, + "EventBodyName": "Council", + "EventDate": "2026-01-08T00:00:00", + "EventTime": "2:00 PM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Final", + "EventLocation": "Council Chambers", + "EventAgendaFile": "https://kansascity.legistar1.com/meetings/19001-agenda.pdf", + "EventMinutesFile": "https://kansascity.legistar1.com/meetings/19001-minutes.pdf", + "EventAgendaLastPublishedUTC": "2026-01-08T17:00:00.000", + "EventMinutesLastPublishedUTC": "2026-01-09T22:00:00.000", + "EventComment": "Webinar Link: https://example.test/council", + "EventVideoPath": null, + "EventMedia": "14001", + "EventInSiteURL": "https://kansascity.legistar.com/MeetingDetail.aspx?LEGID=19001" + }, + { + "EventId": 19002, + "EventGuid": "22222222-2222-4222-8222-222222222222", + "EventLastModifiedUtc": "2026-07-03T20:00:00.000", + "EventRowVersion": "kc-special-v1", + "EventBodyId": 138, + "EventBodyName": "Council", + "EventDate": "2026-07-02T00:00:00", + "EventTime": "12:30 PM", + "EventAgendaStatusName": "Final", + "EventMinutesStatusName": "Draft", + "EventLocation": "KCPD Headquarters - Community Room", + "EventAgendaFile": "https://kansascity.legistar1.com/meetings/19002-agenda.pdf", + "EventMinutesFile": null, + "EventAgendaLastPublishedUTC": "2026-07-02T16:00:00.000", + "EventMinutesLastPublishedUTC": null, + "EventComment": "Special Council Meeting", + "EventVideoPath": "https://videos.kcmo.gov/19002", + "EventMedia": null, + "EventInSiteURL": "https://kansascity.legistar.com/MeetingDetail.aspx?LEGID=19002" + }, + { + "EventId": 19003, + "EventGuid": "33333333-3333-4333-8333-333333333333", + "EventLastModifiedUtc": "2026-02-12T18:00:00.000", + "EventRowVersion": "kc-cancelled-v1", + "EventBodyId": 138, + "EventBodyName": "Council", + "EventDate": "2026-02-12T00:00:00", + "EventTime": "2:00 PM", + "EventAgendaStatusName": "Cancelled", + "EventMinutesStatusName": null, + "EventLocation": "Council Chambers", + "EventAgendaFile": null, + "EventMinutesFile": null, + "EventAgendaLastPublishedUTC": null, + "EventMinutesLastPublishedUTC": null, + "EventComment": "Meeting cancelled", + "EventVideoPath": null, + "EventMedia": null, + "EventInSiteURL": "https://kansascity.legistar.com/MeetingDetail.aspx?LEGID=19003" + }, + { + "EventId": 19004, + "EventGuid": "44444444-4444-4444-8444-444444444444", + "EventLastModifiedUtc": "2026-04-16T21:30:00.000", + "EventRowVersion": "kc-revised-v2", + "EventBodyId": 138, + "EventBodyName": "Council", + "EventDate": "2026-04-16T00:00:00", + "EventTime": "2:00 PM", + "EventAgendaStatusName": "Final-Revised", + "EventMinutesStatusName": "Final-Revised", + "EventLocation": "26th Floor, Council Chambers", + "EventAgendaFile": "https://kansascity.legistar1.com/meetings/19004-agenda-revised.pdf", + "EventMinutesFile": "https://kansascity.legistar1.com/meetings/19004-minutes-revised.pdf", + "EventAgendaLastPublishedUTC": "2026-04-16T20:30:00.000", + "EventMinutesLastPublishedUTC": "2026-04-16T21:30:00.000", + "EventComment": "Webinar Link: https://example.test/council", + "EventVideoPath": null, + "EventMedia": null, + "EventInSiteURL": "https://kansascity.legistar.com/MeetingDetail.aspx?LEGID=19004" + } +] diff --git a/apps/scraper/src/scrapers/fixtures/kansas-city-council/items.json b/apps/scraper/src/scrapers/fixtures/kansas-city-council/items.json new file mode 100644 index 0000000..b24796e --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/kansas-city-council/items.json @@ -0,0 +1,75 @@ +[ + { + "EventItemId": 106909, + "EventItemLastModifiedUtc": "2026-07-16T21:12:40.197", + "EventItemRowVersion": "kc-item-v2", + "EventItemEventId": 19004, + "EventItemAgendaSequence": 15, + "EventItemAgendaNumber": null, + "EventItemVersion": "2", + "EventItemAgendaNote": "Committee substitute", + "EventItemMinutesNote": "The motion carried.", + "EventItemActionId": 409, + "EventItemActionName": "Passed as Substituted", + "EventItemActionText": "A motion was made that this Ordinance be Passed as Substituted.", + "EventItemPassedFlagName": "Pass", + "EventItemRollCallFlag": 0, + "EventItemTitle": "Approving funding for the 1815 Paseo Project.", + "EventItemTally": null, + "EventItemConsent": 0, + "EventItemMover": "Councilmember A", + "EventItemSeconder": "Councilmember B", + "EventItemMatterId": 50729, + "EventItemMatterGuid": "B41F898F-F92D-4A4E-B3B1-33F53A59EA56", + "EventItemMatterFile": "260582", + "EventItemMatterName": null, + "EventItemMatterType": "Ordinance", + "EventItemMatterStatus": "Passed", + "EventItemMatterAttachments": [ + { + "MatterAttachmentId": 104574, + "MatterAttachmentLastModifiedUtc": "2026-07-06T17:39:27.393", + "MatterAttachmentRowVersion": "attachment-v1", + "MatterAttachmentName": "Council Agenda Packet", + "MatterAttachmentHyperlink": "https://kansascity.legistar1.com/attachments/packet.pdf", + "MatterAttachmentShowOnInternetPage": true + }, + { + "MatterAttachmentId": 104575, + "MatterAttachmentLastModifiedUtc": "2026-07-08T15:58:22.030", + "MatterAttachmentRowVersion": "attachment-v1", + "MatterAttachmentName": "Docket Memo", + "MatterAttachmentHyperlink": "https://kansascity.legistar1.com/attachments/docket-memo.pdf", + "MatterAttachmentShowOnInternetPage": true + } + ] + }, + { + "EventItemId": 106906, + "EventItemLastModifiedUtc": "2026-07-16T20:07:18.743", + "EventItemRowVersion": "kc-roll-call-v1", + "EventItemEventId": 19004, + "EventItemAgendaSequence": 4, + "EventItemAgendaNumber": null, + "EventItemVersion": null, + "EventItemAgendaNote": null, + "EventItemMinutesNote": null, + "EventItemActionId": null, + "EventItemActionName": null, + "EventItemActionText": null, + "EventItemPassedFlagName": null, + "EventItemRollCallFlag": 1, + "EventItemTitle": "ROLL CALL:", + "EventItemTally": null, + "EventItemConsent": 0, + "EventItemMover": null, + "EventItemSeconder": null, + "EventItemMatterId": null, + "EventItemMatterGuid": null, + "EventItemMatterFile": null, + "EventItemMatterName": null, + "EventItemMatterType": null, + "EventItemMatterStatus": null, + "EventItemMatterAttachments": [] + } +] diff --git a/apps/scraper/src/scrapers/fixtures/kansas-city-council/votes.json b/apps/scraper/src/scrapers/fixtures/kansas-city-council/votes.json new file mode 100644 index 0000000..b9ac84c --- /dev/null +++ b/apps/scraper/src/scrapers/fixtures/kansas-city-council/votes.json @@ -0,0 +1,20 @@ +[ + { + "VoteId": 89823, + "VoteLastModifiedUtc": "2026-07-16T21:05:22.227", + "VotePersonId": 194, + "VotePersonName": "Quinton Lucas", + "VoteValueName": "Nay", + "VoteSort": 63, + "VoteEventItemId": 106909 + }, + { + "VoteId": 89824, + "VoteLastModifiedUtc": "2026-07-16T21:05:22.237", + "VotePersonId": 196, + "VotePersonName": "Kevin O'Neill", + "VoteValueName": "Aye", + "VoteSort": 64, + "VoteEventItemId": 106909 + } +] diff --git a/apps/scraper/src/scrapers/kansas-city-council.config.ts b/apps/scraper/src/scrapers/kansas-city-council.config.ts new file mode 100644 index 0000000..8eb9128 --- /dev/null +++ b/apps/scraper/src/scrapers/kansas-city-council.config.ts @@ -0,0 +1,11 @@ +import type { ScraperEnvContract } from "@acme/env"; + +export const kansasCityCouncilConfig = { + id: "kansas-city-council", + name: "Kansas City Council", + source: "Kansas City Legistar Web API ā€” Council meetings and roll-call votes", + environment: { + required: ["POSTGRES_URL"], + optional: ["KANSAS_CITY_COUNCIL_MAX_ITEMS"], + }, +} as const satisfies ScraperEnvContract; diff --git a/apps/scraper/src/scrapers/kansas-city-council.test.ts b/apps/scraper/src/scrapers/kansas-city-council.test.ts new file mode 100644 index 0000000..206c60b --- /dev/null +++ b/apps/scraper/src/scrapers/kansas-city-council.test.ts @@ -0,0 +1,112 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + adaptKansasCityItem, + adaptKansasCityMeeting, + adaptKansasCityVote, + currentKansasCityCouncilCycleStart, + isDiscoverableKansasCityEvent, + parseKansasCityStart, +} from "./kansas-city-council.js"; + +async function fixture(name: string): Promise { + const url = new URL( + `./fixtures/kansas-city-council/${name}.json`, + import.meta.url, + ); + return JSON.parse(await readFile(url, "utf8")) as unknown; +} + +void test("maps regular, special, cancelled, and revised Council meetings", async () => { + const events = (await fixture("events")) as unknown[]; + const [regular, special, cancelled, revised] = events.map( + adaptKansasCityMeeting, + ); + + assert.equal(regular?.meetingType, "Regular Meeting"); + assert.match(regular?.videoUrl ?? "", /ID1=14001/); + assert.equal(special?.meetingType, "Special Meeting"); + assert.equal(special?.location, "KCPD Headquarters - Community Room"); + assert.equal(cancelled?.isCancelled, true); + assert.equal(cancelled?.status, "cancelled"); + assert.equal(revised?.isAmended, true); + assert.equal(revised?.documents[0]?.title, "Revised Agenda"); + assert.equal(revised?.documents[1]?.title, "Revised Minutes"); +}); + +void test("maps legislation references, actions, packets, and named votes", async () => { + const [rawItem, rawRollCall] = (await fixture("items")) as unknown[]; + const [rawNay, rawAye] = (await fixture("votes")) as unknown[]; + const item = adaptKansasCityItem(rawItem); + const rollCall = adaptKansasCityItem(rawRollCall); + const nay = adaptKansasCityVote(rawNay); + const aye = adaptKansasCityVote(rawAye); + + assert.equal(item.itemNumber, "260582"); + assert.equal(item.itemType, "Ordinance"); + assert.equal(item.action, "Passed as Substituted"); + assert.equal(item.outcome, "Pass"); + assert.equal(item.shouldFetchVotes, true); + assert.equal(item.documents[0]?.type, "packet"); + assert.equal(item.documents[1]?.type, "attachment"); + assert.equal(rollCall.section, "ROLL CALL"); + assert.equal(rollCall.shouldFetchVotes, true); + assert.deepEqual( + [nay.voterName, nay.value, aye.voterName, aye.value], + ["Quinton Lucas", "Nay", "Kevin O'Neill", "Aye"], + ); +}); + +void test("keeps stable IDs while revisions and document replacements change hashes", async () => { + const [raw] = (await fixture("events")) as Record[]; + const original = adaptKansasCityMeeting(raw); + const replacement = adaptKansasCityMeeting({ + ...raw, + EventAgendaFile: + "https://kansascity.legistar1.com/meetings/19001-agenda-v2.pdf", + EventAgendaLastPublishedUTC: "2026-01-08T19:00:00.000", + EventLastModifiedUtc: "2026-01-08T19:01:00.000", + EventRowVersion: "kc-event-v2", + }); + + assert.equal(original.externalId, replacement.externalId); + assert.notEqual(original.sourceVersion, replacement.sourceVersion); + assert.notEqual(original.contentHash, replacement.contentHash); + assert.notEqual( + original.documents[0]?.checksum, + replacement.documents[0]?.checksum, + ); +}); + +void test("uses Central Time DST and the active four-year Council term", () => { + assert.equal( + parseKansasCityStart("2026-01-08T00:00:00", "2:00 PM").toISOString(), + "2026-01-08T20:00:00.000Z", + ); + assert.equal( + parseKansasCityStart("2026-07-02T00:00:00", "2:00 PM").toISOString(), + "2026-07-02T19:00:00.000Z", + ); + assert.equal( + currentKansasCityCouncilCycleStart( + new Date("2026-07-22T00:00:00Z"), + ).toISOString(), + "2023-08-01T00:00:00.000Z", + ); +}); + +void test("excludes hidden test meetings from primary-body discovery", async () => { + const [raw] = (await fixture("events")) as Record[]; + assert.equal(isDiscoverableKansasCityEvent(raw), true); + assert.equal( + isDiscoverableKansasCityEvent({ + ...raw, + EventId: 99999, + EventAgendaStatusName: "Hidden", + EventComment: "Test Meeting", + }), + false, + ); +}); diff --git a/apps/scraper/src/scrapers/kansas-city-council.ts b/apps/scraper/src/scrapers/kansas-city-council.ts new file mode 100644 index 0000000..5e8957c --- /dev/null +++ b/apps/scraper/src/scrapers/kansas-city-council.ts @@ -0,0 +1,654 @@ +import { createHash } from "node:crypto"; +import pLimit from "p-limit"; +import { z } from "zod/v4"; + +import { and, eq, notInArray } from "@acme/db"; +import { db } from "@acme/db/client"; +import { + LocalGovernmentAgendaItem, + LocalGovernmentDocument, + LocalGovernmentMeeting, + LocalGovernmentVote, +} from "@acme/db/schema"; + +import type { Scraper } from "../utils/types.js"; +import { setExpectedTotal } from "../utils/db/metrics.js"; +import { fetchWithRetry } from "../utils/fetch.js"; +import { createLogger } from "../utils/log.js"; +import { kansasCityCouncilConfig } from "./kansas-city-council.config.js"; + +const API_BASE = "https://webapi.legistar.com/v1/kansascity"; +const SITE_BASE = "https://kansascity.legistar.com"; +const SITE_GROUP = "D2E89A09-8736-4EFB-B4AE-572E0903BD5A"; +const SITE_GROUP_ID = 821; +const PROVIDER = "legistar"; +const JURISDICTION = "kansas-city-mo"; +const COUNCIL_BODY_ID = 138; +const SOURCE_VERSION = "kansas-city-legistar-v1"; +const TIMEZONE = "America/Chicago"; +const DEFAULT_MAX_ITEMS = 250; +const USER_AGENT = + "Billion civic data scraper (+https://github.com/billion-app/billion)"; +const logger = createLogger(kansasCityCouncilConfig.name); + +const attachmentSchema = z + .object({ + MatterAttachmentId: z.number(), + MatterAttachmentLastModifiedUtc: z.string(), + MatterAttachmentRowVersion: z.string(), + MatterAttachmentName: z.string(), + MatterAttachmentHyperlink: z.string().nullable(), + MatterAttachmentShowOnInternetPage: z.boolean().optional(), + }) + .passthrough(); + +export const kansasCityEventSchema = z + .object({ + EventId: z.number(), + EventGuid: z.string(), + EventLastModifiedUtc: z.string(), + EventRowVersion: z.string(), + EventBodyId: z.number(), + EventBodyName: z.string(), + EventDate: z.string(), + EventTime: z.string().nullable(), + EventAgendaStatusName: z.string().nullable(), + EventMinutesStatusName: z.string().nullable(), + EventLocation: z.string().nullable(), + EventAgendaFile: z.string().nullable(), + EventMinutesFile: z.string().nullable(), + EventAgendaLastPublishedUTC: z.string().nullable().optional(), + EventMinutesLastPublishedUTC: z.string().nullable().optional(), + EventComment: z.string().nullable(), + EventVideoPath: z.string().nullable(), + EventMedia: z.union([z.string(), z.number()]).nullable().optional(), + EventInSiteURL: z.string().nullable(), + }) + .passthrough(); + +export const kansasCityItemSchema = z + .object({ + EventItemId: z.number(), + EventItemLastModifiedUtc: z.string(), + EventItemRowVersion: z.string(), + EventItemEventId: z.number(), + EventItemAgendaSequence: z.number(), + EventItemAgendaNumber: z.string().nullable(), + EventItemVersion: z.string().nullable().optional(), + EventItemAgendaNote: z.string().nullable(), + EventItemMinutesNote: z.string().nullable(), + EventItemActionId: z.number().nullable(), + EventItemActionName: z.string().nullable(), + EventItemActionText: z.string().nullable(), + EventItemPassedFlagName: z.string().nullable(), + EventItemRollCallFlag: z.number().nullable(), + EventItemTitle: z.string().nullable(), + EventItemTally: z.string().nullable(), + EventItemConsent: z.number(), + EventItemMover: z.string().nullable(), + EventItemSeconder: z.string().nullable(), + EventItemMatterId: z.number().nullable(), + EventItemMatterGuid: z.string().nullable(), + EventItemMatterFile: z.string().nullable(), + EventItemMatterName: z.string().nullable(), + EventItemMatterType: z.string().nullable(), + EventItemMatterStatus: z.string().nullable(), + EventItemMatterAttachments: z.array(attachmentSchema).nullable(), + }) + .passthrough(); + +export const kansasCityVoteSchema = z + .object({ + VoteId: z.number(), + VoteLastModifiedUtc: z.string(), + VotePersonId: z.number(), + VotePersonName: z.string(), + VoteValueName: z.string(), + VoteSort: z.number(), + VoteEventItemId: z.number(), + }) + .passthrough(); + +type KansasCityEvent = z.infer; +type KansasCityItem = z.infer; +type KansasCityVote = z.infer; +type DocumentType = "agenda" | "packet" | "minutes" | "attachment"; + +interface KansasCityDocument { + type: DocumentType; + title: string; + url: string; + mediaType: string | null; + checksum: string; +} + +interface EnrichedItem { + item: ReturnType; + votes: ReturnType[] | undefined; +} + +function hash(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function datePart(value: string): string { + const match = /^(\d{4}-\d{2}-\d{2})/.exec(value); + if (!match) throw new Error(`Invalid Legistar date: ${value}`); + return match[1]!; +} + +function nthSunday(year: number, monthIndex: number, nth: number): number { + const first = new Date(Date.UTC(year, monthIndex, 1)).getUTCDay(); + return 1 + ((7 - first) % 7) + (nth - 1) * 7; +} + +function centralUtcOffset(date: string): "-05:00" | "-06:00" { + const [year, month, day] = date.split("-").map(Number) as [ + number, + number, + number, + ]; + const dstStart = nthSunday(year, 2, 2); + const dstEnd = nthSunday(year, 10, 1); + const isDst = + (month > 3 && month < 11) || + (month === 3 && day >= dstStart) || + (month === 11 && day < dstEnd); + return isDst ? "-05:00" : "-06:00"; +} + +export function parseKansasCityStart( + eventDate: string, + eventTime: string | null, +): Date { + const date = datePart(eventDate); + const match = eventTime?.trim().match(/^(\d{1,2}):(\d{2})\s*([AP]M)$/i); + let hour = 0; + let minute = 0; + if (match) { + hour = Number(match[1]) % 12; + minute = Number(match[2]); + if (match[3]!.toUpperCase() === "PM") hour += 12; + } + return new Date( + `${date}T${String(hour).padStart(2, "0")}:${String(minute).padStart(2, "0")}:00${centralUtcOffset(date)}`, + ); +} + +/** Kansas City Council terms start August 1 every four years (2023 anchor). */ +export function currentKansasCityCouncilCycleStart(now = new Date()): Date { + const anchorYear = 2023; + const year = now.getUTCFullYear(); + let startYear = anchorYear + Math.floor((year - anchorYear) / 4) * 4; + const candidate = new Date(Date.UTC(startYear, 7, 1)); + if (candidate > now) startYear -= 4; + return new Date(Date.UTC(startYear, 7, 1)); +} + +function mediaType(url: string): string | null { + const extension = new URL(url).pathname.split(".").pop()?.toLowerCase(); + if (extension === "pdf") return "application/pdf"; + if (extension === "doc") return "application/msword"; + if (extension === "docx") + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + if (extension === "pptx") + return "application/vnd.openxmlformats-officedocument.presentationml.presentation"; + return null; +} + +function meetingDocuments( + event: KansasCityEvent, + isAmended: boolean, +): KansasCityDocument[] { + const documents: KansasCityDocument[] = []; + if (event.EventAgendaFile) { + documents.push({ + type: "agenda", + title: `${isAmended ? "Revised " : ""}Agenda`, + url: event.EventAgendaFile, + mediaType: mediaType(event.EventAgendaFile), + checksum: hash({ + url: event.EventAgendaFile, + publishedAt: event.EventAgendaLastPublishedUTC, + rowVersion: event.EventRowVersion, + }), + }); + } + if (event.EventMinutesFile) { + documents.push({ + type: "minutes", + title: /revis/i.test(event.EventMinutesStatusName ?? "") + ? "Revised Minutes" + : "Minutes", + url: event.EventMinutesFile, + mediaType: mediaType(event.EventMinutesFile), + checksum: hash({ + url: event.EventMinutesFile, + publishedAt: event.EventMinutesLastPublishedUTC, + rowVersion: event.EventRowVersion, + }), + }); + } + return documents; +} + +function videoUrl(event: KansasCityEvent): string | null { + if (event.EventVideoPath) { + return new URL(event.EventVideoPath, SITE_BASE).toString(); + } + if (!event.EventMedia) return null; + return `${SITE_BASE}/Video.aspx?Mode=Granicus&ID1=${event.EventMedia}&G=${SITE_GROUP}&Mode2=Video`; +} + +function markerText(event: KansasCityEvent): string { + return [ + event.EventComment, + event.EventAgendaStatusName, + event.EventMinutesStatusName, + event.EventAgendaFile, + event.EventMinutesFile, + ] + .filter(Boolean) + .join(" "); +} + +export function isDiscoverableKansasCityEvent(input: unknown): boolean { + const event = kansasCityEventSchema.parse(input); + return ( + event.EventBodyId === COUNCIL_BODY_ID && + !/hidden/i.test(event.EventAgendaStatusName ?? "") && + !/^test meeting$/i.test(event.EventComment?.trim() ?? "") + ); +} + +export function adaptKansasCityMeeting(input: unknown) { + const event = kansasCityEventSchema.parse(input); + const markers = markerText(event); + const isCancelled = /\bcancel(?:led|ed|lation)?\b/i.test(markers); + const isAmended = /\b(amend(?:ed|ment)?|revis(?:ed|ion)|corrected)\b/i.test( + markers, + ); + const isSpecial = /\bspecial\b/i.test(markers); + const meetingType = isSpecial ? "Special Meeting" : "Regular Meeting"; + const sourceUrl = + event.EventInSiteURL ?? + `${SITE_BASE}/MeetingDetail.aspx?LEGID=${event.EventId}&GID=${SITE_GROUP_ID}&G=${SITE_GROUP}`; + const mapped = { + source: PROVIDER, + jurisdiction: JURISDICTION, + externalId: String(event.EventId), + governingBody: event.EventBodyName, + title: `${event.EventBodyName} ${meetingType}`, + meetingType, + startsAt: parseKansasCityStart(event.EventDate, event.EventTime), + timezone: TIMEZONE, + location: event.EventLocation, + status: isCancelled + ? "cancelled" + : (event.EventMinutesStatusName ?? + event.EventAgendaStatusName ?? + "scheduled"), + isCancelled, + isAmended, + canonicalUrl: sourceUrl, + videoUrl: videoUrl(event), + documents: meetingDocuments(event, isAmended), + sourceVersion: `${SOURCE_VERSION}:${event.EventRowVersion}`, + sourceUpdatedAt: new Date(event.EventLastModifiedUtc), + }; + return { ...mapped, contentHash: hash(mapped) }; +} + +function attachmentDocuments(item: KansasCityItem): KansasCityDocument[] { + return (item.EventItemMatterAttachments ?? []) + .filter( + (attachment) => + attachment.MatterAttachmentHyperlink && + attachment.MatterAttachmentShowOnInternetPage !== false, + ) + .map((attachment) => { + const url = attachment.MatterAttachmentHyperlink!; + const isPacket = /\bagenda\s+packet\b/i.test( + attachment.MatterAttachmentName, + ); + return { + type: isPacket ? ("packet" as const) : ("attachment" as const), + title: attachment.MatterAttachmentName, + url, + mediaType: mediaType(url), + checksum: hash({ + id: attachment.MatterAttachmentId, + rowVersion: attachment.MatterAttachmentRowVersion, + updatedAt: attachment.MatterAttachmentLastModifiedUtc, + url, + }), + }; + }); +} + +export function adaptKansasCityItem(input: unknown) { + const item = kansasCityItemSchema.parse(input); + const title = item.EventItemTitle?.trim() || "Untitled agenda item"; + const isSection = + !item.EventItemMatterId && !item.EventItemActionId && /:\s*$/.test(title); + const mapped = { + externalId: String(item.EventItemId), + meetingExternalId: String(item.EventItemEventId), + sequence: item.EventItemAgendaSequence, + itemNumber: item.EventItemMatterFile ?? item.EventItemAgendaNumber, + section: isSection ? title.replace(/:\s*$/, "") : null, + itemType: + item.EventItemMatterType ?? (isSection ? "section" : "agenda-item"), + title, + description: item.EventItemAgendaNote, + minutesNote: item.EventItemMinutesNote, + consent: item.EventItemConsent === 1, + action: item.EventItemActionName, + motion: item.EventItemActionText, + outcome: item.EventItemPassedFlagName ?? item.EventItemMatterStatus, + voteSummary: item.EventItemTally, + mover: item.EventItemMover, + seconder: item.EventItemSeconder, + matterId: item.EventItemMatterId, + matterGuid: item.EventItemMatterGuid, + documents: attachmentDocuments(item), + shouldFetchVotes: + item.EventItemRollCallFlag === 1 || item.EventItemActionId !== null, + sourceVersion: `${SOURCE_VERSION}:${item.EventItemRowVersion}`, + sourceUpdatedAt: new Date(item.EventItemLastModifiedUtc), + sourceUrl: `${SITE_BASE}/MeetingDetail.aspx?LEGID=${item.EventItemEventId}&GID=${SITE_GROUP_ID}&G=${SITE_GROUP}`, + }; + return { ...mapped, contentHash: hash(mapped) }; +} + +export function adaptKansasCityVote(input: unknown) { + const vote = kansasCityVoteSchema.parse(input); + return { + externalId: String(vote.VoteId), + itemExternalId: String(vote.VoteEventItemId), + voterExternalId: String(vote.VotePersonId), + voterName: vote.VotePersonName, + value: vote.VoteValueName, + sort: vote.VoteSort, + sourceUpdatedAt: new Date(vote.VoteLastModifiedUtc), + }; +} + +function summarizeVotes( + votes: ReturnType[], +): string | null { + if (votes.length === 0) return null; + const counts = new Map(); + for (const vote of votes) { + counts.set(vote.value, (counts.get(vote.value) ?? 0) + 1); + } + return [...counts.entries()] + .map(([value, count]) => `${count} ${value}`) + .join(", "); +} + +async function fetchJson(url: URL): Promise { + const response = await fetchWithRetry(url.toString(), { + headers: { Accept: "application/json", "User-Agent": USER_AGENT }, + timeoutMs: 30_000, + }); + return response.json() as Promise; +} + +async function fetchVotes( + item: ReturnType, +): Promise[] | undefined> { + if (!item.shouldFetchVotes) return undefined; + try { + const url = new URL(`${API_BASE}/EventItems/${item.externalId}/Votes`); + return z + .array(z.unknown()) + .parse(await fetchJson(url)) + .flatMap((raw) => { + const vote = kansasCityVoteSchema.safeParse(raw); + if (!vote.success) { + logger.warn(`Skipping invalid vote for item ${item.externalId}`); + return []; + } + return [adaptKansasCityVote(vote.data)]; + }); + } catch (error) { + logger.warn(`Votes unavailable for item ${item.externalId}`, error); + return undefined; + } +} + +function dedupeDocuments(documents: KansasCityDocument[]) { + return [ + ...new Map(documents.map((document) => [document.url, document])).values(), + ]; +} + +async function persistMeeting( + meeting: ReturnType, + enrichedItems: EnrichedItem[], +): Promise { + const fetchedAt = new Date(); + const documents = dedupeDocuments([ + ...meeting.documents, + ...enrichedItems.flatMap(({ item }) => item.documents), + ]); + const contentHash = hash({ + meeting: meeting.contentHash, + documents: documents.map(({ type, url, checksum }) => ({ + type, + url, + checksum, + })), + items: enrichedItems.map(({ item, votes }) => ({ + hash: item.contentHash, + votes, + })), + }); + const { + documents: _meetingDocuments, + contentHash: _baseHash, + ...meetingRow + } = meeting; + + await db.transaction(async (tx) => { + const [storedMeeting] = await tx + .insert(LocalGovernmentMeeting) + .values({ ...meetingRow, contentHash, fetchedAt }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentMeeting.source, + LocalGovernmentMeeting.jurisdiction, + LocalGovernmentMeeting.externalId, + ], + set: { ...meetingRow, contentHash, fetchedAt, updatedAt: fetchedAt }, + }) + .returning({ id: LocalGovernmentMeeting.id }); + if (!storedMeeting) + throw new Error(`Failed to persist meeting ${meeting.externalId}`); + + await tx + .update(LocalGovernmentDocument) + .set({ isCurrent: false, updatedAt: fetchedAt }) + .where(eq(LocalGovernmentDocument.meetingId, storedMeeting.id)); + + for (const document of documents) { + await tx + .insert(LocalGovernmentDocument) + .values({ + meetingId: storedMeeting.id, + type: document.type, + title: document.title, + url: document.url, + mediaType: document.mediaType, + checksum: document.checksum, + isCurrent: true, + fetchedAt, + }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentDocument.meetingId, + LocalGovernmentDocument.type, + LocalGovernmentDocument.url, + ], + set: { + title: document.title, + mediaType: document.mediaType, + checksum: document.checksum, + isCurrent: true, + fetchedAt, + updatedAt: fetchedAt, + }, + }); + } + + for (const enriched of enrichedItems) { + const { item, votes } = enriched; + const voteSummary = item.voteSummary ?? summarizeVotes(votes ?? []); + const itemRow = { + externalId: item.externalId, + sequence: item.sequence, + itemNumber: item.itemNumber, + section: item.section, + itemType: item.itemType, + title: item.title, + description: item.description, + minutesNote: item.minutesNote, + consent: item.consent, + action: item.action, + motion: item.motion, + outcome: item.outcome, + voteSummary, + mover: item.mover, + seconder: item.seconder, + sourceVersion: item.sourceVersion, + contentHash: item.contentHash, + sourceUpdatedAt: item.sourceUpdatedAt, + sourceUrl: item.sourceUrl, + }; + const [storedItem] = await tx + .insert(LocalGovernmentAgendaItem) + .values({ ...itemRow, meetingId: storedMeeting.id }) + .onConflictDoUpdate({ + target: [ + LocalGovernmentAgendaItem.meetingId, + LocalGovernmentAgendaItem.externalId, + ], + set: { ...itemRow, updatedAt: fetchedAt }, + }) + .returning({ id: LocalGovernmentAgendaItem.id }); + if (!storedItem) + throw new Error(`Failed to persist item ${item.externalId}`); + + // An unavailable vote endpoint preserves the last known roll call. + if (votes === undefined) continue; + await tx + .delete(LocalGovernmentVote) + .where(eq(LocalGovernmentVote.agendaItemId, storedItem.id)); + if (votes.length > 0) { + await tx.insert(LocalGovernmentVote).values( + votes.map((vote) => ({ + agendaItemId: storedItem.id, + externalId: vote.externalId, + voterExternalId: vote.voterExternalId, + voterName: vote.voterName, + value: vote.value, + sort: vote.sort, + sourceUpdatedAt: vote.sourceUpdatedAt, + fetchedAt, + })), + ); + } + } + + const staleItemFilter = + enrichedItems.length === 0 + ? eq(LocalGovernmentAgendaItem.meetingId, storedMeeting.id) + : and( + eq(LocalGovernmentAgendaItem.meetingId, storedMeeting.id), + notInArray( + LocalGovernmentAgendaItem.externalId, + enrichedItems.map(({ item }) => item.externalId), + ), + ); + await tx.delete(LocalGovernmentAgendaItem).where(staleItemFilter); + }); +} + +async function scrapeMeeting(rawEvent: unknown): Promise { + const meeting = adaptKansasCityMeeting(rawEvent); + const itemsUrl = new URL( + `${API_BASE}/Events/${meeting.externalId}/EventItems`, + ); + itemsUrl.searchParams.set("AgendaNote", "1"); + itemsUrl.searchParams.set("MinutesNote", "1"); + itemsUrl.searchParams.set("Attachments", "1"); + const rawItems = z.array(z.unknown()).parse(await fetchJson(itemsUrl)); + const items = rawItems.flatMap((rawItem) => { + const parsed = kansasCityItemSchema.safeParse(rawItem); + if (!parsed.success) { + logger.warn(`Skipping invalid item for meeting ${meeting.externalId}`); + return []; + } + return [adaptKansasCityItem(parsed.data)]; + }); + + // Kansas City stores votes on actioned items even when RollCallFlag is zero. + // Two concurrent vote requests keeps pressure on the public host conservative. + const voteLimit = pLimit(2); + const enrichedItems = await Promise.all( + items.map(async (item) => ({ + item, + votes: await voteLimit(() => fetchVotes(item)), + })), + ); + await persistMeeting(meeting, enrichedItems); + logger.success(`Synced ${meeting.title} (${meeting.externalId})`); +} + +async function scrape(maxItems = DEFAULT_MAX_ITEMS): Promise { + const start = currentKansasCityCouncilCycleStart(); + const end = new Date(Date.UTC(start.getUTCFullYear() + 4, 7, 1)); + const eventsUrl = new URL(`${API_BASE}/Events`); + eventsUrl.searchParams.set( + "$filter", + `EventDate ge datetime'${start.toISOString().slice(0, 10)}' and EventDate lt datetime'${end.toISOString().slice(0, 10)}' and EventBodyId eq ${COUNCIL_BODY_ID}`, + ); + eventsUrl.searchParams.set("$orderby", "EventDate asc"); + eventsUrl.searchParams.set("$top", String(maxItems)); + + logger.info( + `Syncing Council term from ${start.toISOString().slice(0, 10)} (structured source; no archive backfill)`, + ); + const rawEvents = z + .array(z.unknown()) + .parse(await fetchJson(eventsUrl)) + .filter(isDiscoverableKansasCityEvent) + .slice(0, maxItems); + setExpectedTotal(rawEvents.length); + + // Meetings are intentionally sequential. Only vote lookups within one + // meeting run concurrently, capped above at two requests. + for (const rawEvent of rawEvents) { + try { + await scrapeMeeting(rawEvent); + } catch (error) { + const parsed = kansasCityEventSchema.safeParse(rawEvent); + logger.error( + `Meeting ${parsed.success ? parsed.data.EventId : "unknown"} failed without aborting run`, + error, + ); + } + } + logger.success(`Completed ${rawEvents.length} Kansas City Council meetings`); +} + +export const kansasCityCouncil: Scraper = { + ...kansasCityCouncilConfig, + scrape: (options) => + scrape( + (options?.maxItems ?? + Number(process.env.KANSAS_CITY_COUNCIL_MAX_ITEMS)) || + DEFAULT_MAX_ITEMS, + ), +}; diff --git a/docs/data-layer.md b/docs/data-layer.md index 0190b2f..daf8e58 100644 --- a/docs/data-layer.md +++ b/docs/data-layer.md @@ -80,9 +80,10 @@ All three content tables share a common pattern: **Provider-neutral local government** ā€” `local_government_meeting`, `local_government_document`, `local_government_agenda_item`, and -`local_government_vote`. These hold systems that do not expose Legistar -semantics, including Durham's OnBase instance and Cedar Park's CivicEngage -source. Meetings are keyed by `(source, jurisdiction, external_id)`; documents +`local_government_vote`. These form the provider-neutral scheduled-source +contract for Durham's OnBase/Legistar sources, Cedar Park's CivicEngage source, +and Kansas City's current-term Council Legistar source. Meetings are keyed by +`(source, jurisdiction, external_id)`; documents are versionable rows and agenda items carry official motion, outcome, and vote text. The API reads persisted rows instead of scraping during a user request. diff --git a/docs/data-sources-api.md b/docs/data-sources-api.md index 2bb929b..11c2459 100644 --- a/docs/data-sources-api.md +++ b/docs/data-sources-api.md @@ -179,11 +179,13 @@ const legislators = await getLegislators({ state: "ca" }); **Sources:** persisted local-government records plus the Legistar Web API **Entry points:** `packages/api/src/lib/local-government.ts`, `packages/api/src/integrations/legistar.ts` **Auth:** None (public API) -**Persisted jurisdiction:** Cedar Park (`cedar-park-tx`); live Legistar jurisdictions: San Jose, Santa Clara County, Sunnyvale +**Persisted jurisdictions:** Cedar Park (`cedar-park-tx`), Durham County +(`durham-county-nc`), and Kansas City (`kansas-city-mo`); live provider-specific +Legistar jurisdictions: San Jose, Santa Clara County, Sunnyvale -The product-facing reader uses normalized persisted records. Cedar Park's -scheduled scraper populates these from the city's official CivicEngage page and -embedded Municode Meetings publication: +The product-facing reader uses normalized persisted records. Scheduled source +adapters populate these from official systems; Kansas City uses the structured +Legistar feed for current-term Council body `138`: ```ts import { @@ -192,7 +194,7 @@ import { } from "@acme/api"; const meetings = await getLocalGovernmentMeetings({ - jurisdiction: "cedar-park-tx", + jurisdiction: "kansas-city-mo", limit: 20, }); const detail = meetings[0] @@ -233,10 +235,10 @@ To add a city: add its `*.legistar.com` subdomain to `JURISDICTIONS` in `integra - **Reader:** `packages/api/src/lib/local-government.ts` - **tRPC:** `localGovernment.meetings`, `localGovernment.meeting` -The Durham and Cedar Park scrapers persist meetings, agenda or minutes item -outlines, supporting-document URLs, and official action/vote text. The reader -is provider-neutral and serves only cached database records; it does not -contact upstream meeting systems in the request path. +The Durham, Cedar Park, and Kansas City scrapers persist meetings, agenda or +minutes item outlines, supporting-document URLs, and official action/vote text. +The reader is provider-neutral and serves only cached database records; it does +not contact upstream meeting systems in the request path. ```ts const meetings = await getLocalGovernmentMeetings({ diff --git a/docs/scraper.md b/docs/scraper.md index 7111888..993a390 100644 --- a/docs/scraper.md +++ b/docs/scraper.md @@ -30,6 +30,7 @@ process environment at runtime, not embedded during the build. | `civicengage.ts` | Cedar Park official council page | local-government tables | CivicEngage entry page + Municode embed; deterministic HTML/PDF parsing | | `durham-onbase.ts` | Durham OnBase Agenda Online | local-government tables | current-cycle meetings, items, attachments, and official actions | | `durham-bocc.ts` | Durham County Legistar API | local-government tables | current-cycle meetings, items, actions, votes, and documents | +| `kansas-city-council.ts` | Kansas City Legistar API | local-government tables | current-term Council meetings, legislation, documents, and named votes | All HTTP goes through one `fetchWithRetry()` utility (`apps/scraper/src/utils/fetch.ts`): exponential backoff (1s/2s/4sā€¦), `Retry-After` support (seconds or HTTP-date), 30s default timeout via `AbortController`, retriable on 429/5xx and `ECONNRESET`/`ECONNREFUSED`, plus a stateful **per-host backoff** that ramps on 429/5xx and relaxes on success. @@ -63,6 +64,28 @@ product does not expose historical election cycles or run an OnBase backfill. Run it with `pnpm --filter @acme/scraper run start durham-bocc`. The default cap is 100 meetings and can be changed with `DURHAM_BOCC_MAX_ITEMS` or `--max-items`. +### Kansas City Council + +`kansas-city-council` reads the official `kansascity` Legistar Web API and +limits discovery to body ID `138` (`Council`, Primary Legislative Body) and the +active four-year Council term, anchored at August 1, 2023. Hidden test rows are +excluded. Meetings are processed sequentially; at most two vote requests run +concurrently and all requests use the shared retry/backoff client. + +Stable Event, EventItem, Matter, attachment, and Vote identifiers feed the +existing provider-neutral local-government tables. The adapter records Central +Time with DST, locations, cancellations, revised publication states, official +agenda/minutes/packet-or-attachment links, video media IDs, legislation file +numbers, actions/outcomes, and named votes. Kansas City publishes votes on some +actioned items even when its roll-call flag is zero, so those items are checked +too. Document descriptor checksums include the official row version/publication +timestamp; replaced URLs make old documents non-current. No PDF OCR, AI, or +archive backfill is used, and no schema migration is required. + +Run `pnpm --filter @acme/scraper run start kansas-city-council`. The default +current-term cap is 250 meetings and can be changed with +`KANSAS_CITY_COUNCIL_MAX_ITEMS` or `--max-items`. + ## Upsert + Change Detection `apps/scraper/src/utils/db/operations.ts` centralizes writes behind a discriminated-union `upsertContent(type, data)` (`type` āˆˆ bill | government_content | court_case). Each run: diff --git a/packages/env/src/registry.ts b/packages/env/src/registry.ts index 472daf5..5a73588 100644 --- a/packages/env/src/registry.ts +++ b/packages/env/src/registry.ts @@ -104,6 +104,11 @@ const scraperSourceLimitDefinitions = [ "24", ], ["DURHAM_BOCC_MAX_ITEMS", "Durham County BOCC meetings per run.", "100"], + [ + "KANSAS_CITY_COUNCIL_MAX_ITEMS", + "Kansas City Council meetings per current-term run.", + "250", + ], ] as const; export const envRegistry = [