billion-app · ThatXliner · Jul 22, 2026
diff --git a/apps/scraper/src/scraper-contracts.ts b/apps/scraper/src/scraper-contracts.ts
@@ -2,6 +2,7 @@ import type { ScraperEnvContract } from "@acme/env";
 
 import { caSosStatementsConfig } from "./scrapers/ca-sos-statements.config.js";
 import { congressConfig } from "./scrapers/congress.config.js";
+import { durhamOnBaseConfig } from "./scrapers/durham-onbase.config.js";
 import { federalregisterConfig } from "./scrapers/federalregister.config.js";
 import { sccCvigConfig } from "./scrapers/scc-cvig.config.js";
 import { scotusConfig } from "./scrapers/scotus.config.js";
@@ -12,4 +13,5 @@ export const scraperContracts: readonly ScraperEnvContract[] = [
   scotusConfig,
   sccCvigConfig,
   caSosStatementsConfig,
+  durhamOnBaseConfig,
 ];
diff --git a/apps/scraper/src/scrapers.ts b/apps/scraper/src/scrapers.ts
@@ -1,6 +1,7 @@
 import type { Scraper } from "./utils/types.js";
 import { caSosStatements } from "./scrapers/ca-sos-statements.js";
 import { congress } from "./scrapers/congress.js";
+import { durhamOnBase } from "./scrapers/durham-onbase.js";
 import { federalregister } from "./scrapers/federalregister.js";
 import { sccCvig } from "./scrapers/scc-cvig.js";
 import { scotus } from "./scrapers/scotus.js";
@@ -11,4 +12,5 @@ export const scrapers: readonly Scraper[] = [
   scotus,
   sccCvig,
   caSosStatements,
+  durhamOnBase,
 ];
diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-agenda.html
@@ -0,0 +1,6 @@
+<html><body>
+<table><tr><td><a href="javascript:loadAgendaItem(21943,true);">Consent Agenda</a></td></tr></table>
+<table><tr><td><a href="javascript:loadAgendaItem(47768,false);">1. Approval of City Council Minutes</a></td></tr><tr><td>To approve the listed minutes. <b>[Approved by Vote: 7/0]</b></td></tr></table>
+<table><tr><td><a href="javascript:loadAgendaItem(21948,true);">General Business Agenda</a></td></tr></table>
+<table><tr><td><a href="javascript:loadAgendaItem(47875,false);">21. Consolidated Annexation – 4802 Cheek Road</a></td></tr><tr><td>Motion 1: To adopt the ordinance. <b>[FAILED by Vote: 0/7]</b> <b>Ayes: None.</b> <b>Nays: Mayor Williams, Council Member Baker.</b> Motion 2: Consider rezoning. <b>No vote taken.</b></td></tr></table>
+</body></html>
diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-index.html
@@ -0,0 +1,3 @@
+<!doctype html><html><body><script>
+showSearchResults(new SearchResults({"MeetingTypes":[],"Meetings":[{"ID":748,"Name":"May 18, 2026 City Council Meeting","MeetingTypeName":"City Council Meeting Agenda","IsAgendaAvailable":true,"IsMinutesAvailable":false,"Time":"2026-05-18T19:00:00-04:00","Location":"101 City Hall Plaza","AgendaUniqueName":"May_18_2026_Agenda","MinutesUniqueName":"May_18_2026_Minutes","LatestDocumentType":1},{"ID":700,"Name":"December 1, 2025 City Council Meeting","MeetingTypeName":"City Council Meeting Agenda","IsAgendaAvailable":true,"IsMinutesAvailable":true,"Time":"2025-12-01T19:00:00-05:00","Location":null,"AgendaUniqueName":"December_1_2025_Agenda","MinutesUniqueName":"December_1_2025_Minutes","LatestDocumentType":2}]}));
+</script></body></html>
diff --git a/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html b/apps/scraper/src/scrapers/__fixtures__/durham-onbase-item.html
@@ -0,0 +1,4 @@
+<html><body><h2>Supporting Documents</h2>
+<a href="/OnBaseAgendaOnline/Documents/DownloadFile/Memo.pdf?documentType=1&amp;meetingId=748&amp;itemId=47768&amp;publishId=272813&amp;isAttachment=True" id="lnkAttachment_867029">Final-Published Attachment - Approval Memo</a>
+<a href="/OnBaseAgendaOnline/Documents/DownloadFile/Minutes.pdf?documentType=1&amp;meetingId=748&amp;itemId=47768&amp;publishId=272814&amp;isAttachment=True" id="lnkAttachment_867030">March 16 City Council Minutes</a>
+</body></html>
diff --git a/apps/scraper/src/scrapers/durham-onbase-parser.test.ts b/apps/scraper/src/scrapers/durham-onbase-parser.test.ts
@@ -0,0 +1,73 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import { describe, it } from "node:test";
+
+import {
+  parseAgendaOutline,
+  parseItemAttachments,
+  parseMeetingIndex,
+} from "./durham-onbase-parser.js";
+
+const fixture = (name: string) =>
+  readFile(new URL(`./__fixtures__/${name}`, import.meta.url), "utf8");
+
+describe("Durham OnBase deterministic parsers", () => {
+  it("parses the embedded meeting index JSON and preserves timezone", async () => {
+    const meetings = parseMeetingIndex(
+      await fixture("durham-onbase-index.html"),
+    );
+    assert.equal(meetings.length, 2);
+    assert.deepEqual(
+      {
+        id: meetings[0]?.id,
+        type: meetings[0]?.meetingType,
+        iso: meetings[0]?.date.toISOString(),
+        latestDocumentType: meetings[1]?.latestDocumentType,
+      },
+      {
+        id: 748,
+        type: "City Council Meeting Agenda",
+        iso: "2026-05-18T23:00:00.000Z",
+        latestDocumentType: 2,
+      },
+    );
+  });
+
+  it("parses sections, items, action text, and vote text", async () => {
+    const items = parseAgendaOutline(
+      await fixture("durham-onbase-agenda.html"),
+    );
+    assert.equal(items.length, 2);
+    assert.deepEqual(
+      {
+        externalId: items[0]?.externalId,
+        section: items[0]?.section,
+        number: items[0]?.agendaNumber,
+        title: items[0]?.title,
+        vote: items[0]?.voteText,
+      },
+      {
+        externalId: "47768",
+        section: "Consent Agenda",
+        number: "1",
+        title: "Approval of City Council Minutes",
+        vote: "[Approved by Vote: 7/0]",
+      },
+    );
+    assert.match(items[1]?.voteText ?? "", /FAILED by Vote: 0\/7/i);
+    assert.match(items[1]?.voteText ?? "", /No vote taken/i);
+  });
+
+  it("parses stable attachment IDs and absolute official URLs", async () => {
+    const attachments = parseItemAttachments(
+      await fixture("durham-onbase-item.html"),
+    );
+    assert.equal(attachments.length, 2);
+    assert.equal(attachments[0]?.externalId, "272813");
+    assert.equal(
+      new URL(attachments[0]!.url).hostname,
+      "cityordinances.durhamnc.gov",
+    );
+    assert.equal(attachments[1]?.title, "March 16 City Council Minutes");
+  });
+});
diff --git a/apps/scraper/src/scrapers/durham-onbase-parser.ts b/apps/scraper/src/scrapers/durham-onbase-parser.ts
@@ -0,0 +1,198 @@
+import * as cheerio from "cheerio";
+
+export const DURHAM_ONBASE_BASE_URL =
+  "https://cityordinances.durhamnc.gov/OnBaseAgendaOnline/";
+
+export interface OnBaseMeetingIndexItem {
+  id: number;
+  name: string;
+  meetingType: string;
+  date: Date;
+  location?: string;
+  isAgendaAvailable: boolean;
+  isMinutesAvailable: boolean;
+  agendaUniqueName: string;
+  minutesUniqueName: string;
+  latestDocumentType: 1 | 2 | 3;
+}
+
+export interface ParsedOnBaseAttachment {
+  externalId: string;
+  title: string;
+  url: string;
+}
+
+export interface ParsedOnBaseAgendaItem {
+  externalId: string;
+  section?: string;
+  agendaNumber?: string;
+  title: string;
+  actionText?: string;
+  voteText?: string;
+  attachments: ParsedOnBaseAttachment[];
+  sortOrder: number;
+}
+
+interface RawMeeting {
+  ID?: unknown;
+  Name?: unknown;
+  MeetingTypeName?: unknown;
+  Time?: unknown;
+  Location?: unknown;
+  IsAgendaAvailable?: unknown;
+  IsMinutesAvailable?: unknown;
+  AgendaUniqueName?: unknown;
+  MinutesUniqueName?: unknown;
+  LatestDocumentType?: unknown;
+}
+
+function cleanText(value: string): string {
+  return value
+    .replace(/\u00a0/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function extractJsonObject(source: string, start: number): string {
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+
+  for (let index = start; index < source.length; index++) {
+    const char = source[index]!;
+    if (inString) {
+      if (escaped) escaped = false;
+      else if (char === "\\") escaped = true;
+      else if (char === '"') inString = false;
+      continue;
+    }
+    if (char === '"') inString = true;
+    else if (char === "{") depth++;
+    else if (char === "}" && --depth === 0)
+      return source.slice(start, index + 1);
+  }
+
+  throw new Error("Unterminated OnBase meeting index JSON");
+}
+
+export function parseMeetingIndex(html: string): OnBaseMeetingIndexItem[] {
+  const marker = "showSearchResults(new SearchResults(";
+  const markerIndex = html.indexOf(marker);
+  if (markerIndex < 0)
+    throw new Error("OnBase meeting index payload not found");
+  const jsonStart = html.indexOf("{", markerIndex + marker.length);
+  if (jsonStart < 0) throw new Error("OnBase meeting index JSON not found");
+
+  const payload = JSON.parse(extractJsonObject(html, jsonStart)) as {
+    Meetings?: RawMeeting[];
+  };
+
+  return (payload.Meetings ?? []).flatMap((raw) => {
+    if (
+      typeof raw.ID !== "number" ||
+      typeof raw.Name !== "string" ||
+      typeof raw.MeetingTypeName !== "string" ||
+      typeof raw.Time !== "string" ||
+      typeof raw.AgendaUniqueName !== "string" ||
+      typeof raw.MinutesUniqueName !== "string"
+    ) {
+      return [];
+    }
+    const date = new Date(raw.Time);
+    if (Number.isNaN(date.getTime())) return [];
+    const latest =
+      raw.LatestDocumentType === 2 || raw.LatestDocumentType === 3
+        ? raw.LatestDocumentType
+        : 1;
+
+    return [
+      {
+        id: raw.ID,
+        name: cleanText(raw.Name),
+        meetingType: cleanText(raw.MeetingTypeName),
+        date,
+        location:
+          typeof raw.Location === "string" && cleanText(raw.Location)
+            ? cleanText(raw.Location)
+            : undefined,
+        isAgendaAvailable: raw.IsAgendaAvailable === true,
+        isMinutesAvailable: raw.IsMinutesAvailable === true,
+        agendaUniqueName: raw.AgendaUniqueName,
+        minutesUniqueName: raw.MinutesUniqueName,
+        latestDocumentType: latest,
+      },
+    ];
+  });
+}
+
+function extractVoteText(actionText: string): string | undefined {
+  const matches = actionText.match(
+    /\[(?:approved|failed)[^\]]*\]|\[motion referred[^\]]*\]|no vote (?:was )?taken\.?|ayes:\s*[^.]*\.?|nays:\s*[^.]*\.?/gi,
+  );
+  return matches ? cleanText(matches.join(" ")) : undefined;
+}
+
+export function parseAgendaOutline(html: string): ParsedOnBaseAgendaItem[] {
+  const $ = cheerio.load(html);
+  const items: ParsedOnBaseAgendaItem[] = [];
+  let currentSection: string | undefined;
+
+  $("a[href*='loadAgendaItem']").each((_, element) => {
+    const anchor = $(element);
+    const href = anchor.attr("href") ?? "";
+    const match = /loadAgendaItem\((\d+),(true|false)\)/.exec(href);
+    if (!match) return;
+    const title = cleanText(anchor.text());
+    if (!title) return;
+    if (match[2] === "true") {
+      currentSection = title;
+      return;
+    }
+
+    const externalId = match[1]!;
+    const agendaMatch = /^(\d+[A-Za-z]?)\.\s*(.+)$/.exec(title);
+    const displayTitle = agendaMatch?.[2] ?? title;
+    const tableText = cleanText(anchor.closest("table").text());
+    const actionText = cleanText(
+      tableText.startsWith(title) ? tableText.slice(title.length) : tableText,
+    );
+
+    items.push({
+      externalId,
+      section: currentSection,
+      agendaNumber: agendaMatch?.[1],
+      title: displayTitle,
+      actionText: actionText || undefined,
+      voteText: actionText ? extractVoteText(actionText) : undefined,
+      attachments: [],
+      sortOrder: items.length,
+    });
+  });
+
+  return items;
+}
+
+export function parseItemAttachments(
+  html: string,
+  baseUrl = DURHAM_ONBASE_BASE_URL,
+): ParsedOnBaseAttachment[] {
+  const $ = cheerio.load(html);
+  return $("a[href*='/Documents/DownloadFile/']")
+    .toArray()
+    .flatMap((element) => {
+      const anchor = $(element);
+      const href = anchor.attr("href");
+      if (!href) return [];
+      const url = new URL(href, baseUrl);
+      const externalId =
+        url.searchParams.get("publishId") ??
+        (anchor.attr("id")?.match(/(\d+)$/)?.[1] || url.toString());
+      return [
+        {
+          externalId,
+          title: cleanText(anchor.text()),
+          url: url.toString(),
+        },
+      ];
+    });
+}
diff --git a/apps/scraper/src/scrapers/durham-onbase.config.ts b/apps/scraper/src/scrapers/durham-onbase.config.ts
@@ -0,0 +1,11 @@
+import type { ScraperEnvContract } from "@acme/env";
+
+export const durhamOnBaseConfig = {
+  id: "durham-onbase",
+  name: "Durham City Council OnBase",
+  source: "City of Durham OnBase Agenda Online",
+  environment: {
+    required: ["POSTGRES_URL"],
+    optional: ["DURHAM_ONBASE_MAX_ITEMS", "DURHAM_ONBASE_CACHE_TTL_HOURS"],
+  },
+} as const satisfies ScraperEnvContract;