fix(get): auto-paginate list endpoints; surface meta + HTTP status/body on errors

src/api/ctClient.ts

@@ -37,6 +37,33 @@ export class CtApiError extends Error {
 
 type Json = Record<string, unknown>;
 
+/**
+ * ChurchTools' list-endpoint pagination envelope, carried as `meta.pagination`
+ * alongside `data`. Field names confirmed against the live API (#50): a page
+ * is exhausted once `current >= lastPage`.
+ */
+export interface CtPagination {
+  total?: number;
+  current?: number;
+  lastPage?: number;
+  limit?: number;
+  count?: number;
+}
+
+export interface CtMeta {
+  pagination?: CtPagination;
+  [key: string]: unknown;
+}
+
+export interface CtPage<T> {
+  data: T[];
+  meta?: CtMeta;
+}
+
+/** Hard stop so a malformed/adversarial pagination response can't loop forever. */
+const MAX_PAGES = 1000;
+const DEFAULT_PAGE_LIMIT = 100;
+
 export class CtClient {
   private cookie: string | null = null;
   private csrfToken: string | null = null;
@@ -108,6 +135,61 @@ export class CtClient {
   }
 
   async request<T = unknown>(method: string, path: string, body?: Json): Promise<T> {
+    const parsed = await this.requestEnvelope(method, path, body);
+    if (parsed === undefined) {
+      return undefined as T;
+    }
+    const envelope = parsed as { data?: T };
+    return (envelope.data ?? envelope) as T;
+  }
+
+  /**
+   * Fetch every page of a ChurchTools list endpoint and concatenate them, so
+   * callers see the whole collection instead of just CT's default first page
+   * (#50). CT caps `limit` at a per-endpoint maximum below 500 on real
+   * instances, so this defaults to a conservative page size and pages via
+   * `?page=N&limit=M` until `meta.pagination.current >= lastPage`. Endpoints
+   * that don't return pagination meta (or return everything on page 1) fall
+   * out after a single request.
+   */
+  async getAll<T = unknown>(path: string, options: { limit?: number } = {}): Promise<CtPage<T>> {
+    const limit = options.limit ?? DEFAULT_PAGE_LIMIT;
+    const items: T[] = [];
+    let meta: CtMeta | undefined;
+    let page = 1;
+    for (let i = 0; i < MAX_PAGES; i++) {
+      const parsed = await this.requestEnvelope("GET", withPageParams(path, page, limit));
+      if (parsed === undefined) {
+        break;
+      }
+      const isArrayEnvelope = Array.isArray(parsed);
+      const envelope = isArrayEnvelope ? undefined : (parsed as { data?: unknown; meta?: CtMeta });
+      const pageData = isArrayEnvelope ? parsed : (envelope?.data ?? parsed);
+      const pageItems = Array.isArray(pageData) ? (pageData as T[]) : [];
+      items.push(...pageItems);
+      const pageMeta = envelope?.meta;
+      meta = pageMeta ?? meta;
+      const pagination = pageMeta?.pagination;
+      if (pageItems.length === 0 || !pagination || pagination.current === undefined || pagination.lastPage === undefined) {
+        break;
+      }
+      if (pagination.current >= pagination.lastPage) {
+        break;
+      }
+      page += 1;
+    }
+    return { data: items, meta };
+  }
+
+  /**
+   * Shared fetch + parse for {@link request} and {@link getAll}: performs the
+   * HTTP call, throws a status/body-carrying {@link CtApiError} on failure,
+   * and returns the raw parsed JSON envelope (still carrying `data`/`meta`) —
+   * or `undefined` for an empty 2xx body. Kept private so `request()`'s
+   * `.data ?? envelope` unwrap stays the single source of truth for existing
+   * callers (plan/apply/adopt) while `getAll()` gets at `meta` too.
+   */
+  private async requestEnvelope(method: string, path: string, body?: Json): Promise<unknown> {
     if (!this.cookie) {
       throw new CtApiError("Not authenticated — run `ct auth login` first", 401, null);
     }
@@ -138,22 +220,20 @@ export class CtClient {
       throw new CtApiError(`${method} ${path} failed`, res.status, await safeBody(res));
     }
     if (res.status === 204) {
-      return undefined as T;
+      return undefined;
     }
     // Any 2xx may carry an empty or non-JSON body (DELETEs commonly do). A bare
     // res.json() there throws a raw SyntaxError naming no request. Read the text
     // first: empty → undefined; unparseable → a CtApiError that names method+path.
     const text = await res.text();
     if (text.trim() === "") {
-      return undefined as T;
+      return undefined;
     }
-    let parsed: { data?: T };
     try {
-      parsed = JSON.parse(text) as { data?: T };
+      return JSON.parse(text);
     } catch {
       throw new CtApiError(`${method} ${path} returned a non-JSON body`, res.status, text);
     }
-    return (parsed.data ?? parsed) as T;
   }
 
   private async refreshCsrfToken(): Promise<void> {
@@ -198,3 +278,9 @@ async function safeBody(res: Response): Promise<unknown> {
     return null;
   }
 }
+
+/** Append `page`/`limit` query params, respecting any query string the caller already has. */
+function withPageParams(path: string, page: number, limit: number): string {
+  const separator = path.includes("?") ? "&" : "?";
+  return `${path}${separator}page=${page}&limit=${limit}`;
+}

src/commands/get.ts

@@ -1,38 +1,63 @@
 import { Command } from "commander";
 import { authedSession } from "../api/session.js";
 import { CATALOG } from "../permissions/catalog.js";
-import { out } from "../ui.js";
+import { info, out } from "../ui.js";
+
+interface ResourceSpec {
+  path: string;
+  /**
+   * Whether this endpoint returns a paged list (auto-paginate through every
+   * page) vs a single object (`whoami`, `info`, the global permissions blob)
+   * where paging params don't apply. Defaults to true.
+   */
+  paginated?: boolean;
+}
 
 /**
  * Read-only imperative queries — immediately useful before any declarative
  * engine exists. Resource → API path map. Paths confirmed against the live
  * spec by the Phase 0 spike (#2, CT 3.123.0); see docs/api-coverage.md.
+ *
+ * List endpoints are auto-paginated (#50): ChurchTools returns only its
+ * default page (10 items) per request, so `ct get groups` on an instance with
+ * 300+ groups silently returned just the first 10 before this fix.
  */
-const RESOURCE_PATHS: Record<string, string> = {
-  whoami: "/whoami",
-  info: "/info",
-  campuses: "/campuses",
-  groups: "/groups",
-  "group-hierarchies": "/groups/hierarchies",
-  "group-types": "/group/grouptypes",
-  "group-roles": "/group/roles",
-  "age-groups": "/group/agegroups",
-  "target-groups": "/group/targetgroups",
-  "dynamic-groups": "/dynamicgroups",
-  "relationship-types": "/person/relationshiptypes",
-  permissions: "/permissions/global",
+const RESOURCE_PATHS: Record<string, ResourceSpec> = {
+  whoami: { path: "/whoami", paginated: false },
+  info: { path: "/info", paginated: false },
+  campuses: { path: "/campuses" },
+  groups: { path: "/groups" },
+  "group-hierarchies": { path: "/groups/hierarchies" },
+  "group-types": { path: "/group/grouptypes" },
+  "group-roles": { path: "/group/roles" },
+  "age-groups": { path: "/group/agegroups" },
+  "target-groups": { path: "/group/targetgroups" },
+  "dynamic-groups": { path: "/dynamicgroups" },
+  "relationship-types": { path: "/person/relationshiptypes" },
+  permissions: { path: "/permissions/global", paginated: false },
 };
 
 export function getCommand(): Command {
   const cmd = new Command("get").description("Read structure resources from ChurchTools (JSON to stdout)");
 
-  for (const [name, path] of Object.entries(RESOURCE_PATHS)) {
+  for (const [name, spec] of Object.entries(RESOURCE_PATHS)) {
     cmd
       .command(name)
-      .description(`GET ${path}`)
+      .description(`GET ${spec.path}`)
       .action(async () => {
         const { client } = await authedSession();
-        out(await client.get(path));
+        if (spec.paginated === false) {
+          out(await client.get(spec.path));
+          return;
+        }
+        const { data, meta } = await client.getAll(spec.path);
+        out(data);
+        const total = meta?.pagination?.total;
+        if (total !== undefined && total !== data.length) {
+          info(`${data.length} of ${total} total`);
+        } else {
+          info(`${data.length} total`);
+        }
       });
   }
 

src/index.ts

@@ -9,7 +9,7 @@ import { applyCommand } from "./commands/apply.js";
 import { destroyCommand } from "./commands/destroy.js";
 import { plannedCommands } from "./commands/placeholders.js";
 import { isMainModule } from "./isMain.js";
-import { error } from "./ui.js";
+import { error, formatError } from "./ui.js";
 
 export function buildProgram(): Command {
   const program = new Command();
@@ -40,7 +40,7 @@ async function main(): Promise<void> {
   try {
     await program.parseAsync(process.argv);
   } catch (err) {
-    error(err instanceof Error ? err.message : String(err));
+    error(formatError(err));
     process.exitCode = 1;
   }
 }

src/ui.ts

@@ -2,6 +2,35 @@
  * Tiny terminal output helpers. Kept dependency-light on purpose.
  */
 import pc from "picocolors";
+import { CtApiError } from "./api/ctClient.js";
+
+/** Response bodies beyond this are truncated so a huge HTML/JSON dump doesn't flood the terminal. */
+const MAX_BODY_CHARS = 2000;
+
+function formatBody(body: unknown): string {
+  if (body === null || body === undefined) {
+    return "";
+  }
+  const text = typeof body === "string" ? body : JSON.stringify(body, null, 2);
+  if (text.length > MAX_BODY_CHARS) {
+    return `${text.slice(0, MAX_BODY_CHARS)}\n… (truncated, ${text.length} chars total)`;
+  }
+  return text;
+}
+
+/**
+ * Render a caught error for the terminal. For {@link CtApiError} this surfaces
+ * the HTTP status + response body — without it, a failing `ct get raw` (or any
+ * API call) prints only "✗ GET ... failed" with no way to see what ChurchTools
+ * actually said (#50).
+ */
+export function formatError(err: unknown): string {
+  if (err instanceof CtApiError) {
+    const body = formatBody(err.body);
+    return `${err.message} (HTTP ${err.status})${body ? `\n${body}` : ""}`;
+  }
+  return err instanceof Error ? err.message : String(err);
+}
 
 export function info(message: string): void {
   process.stderr.write(`${message}\n`);

tests/ctClient.test.ts

@@ -99,4 +99,78 @@ describe("CtClient", () => {
       message: expect.stringContaining("PUT /campuses/0"),
     });
   });
+
+  describe("getAll (#50)", () => {
+    it("auto-paginates until lastPage is reached, concatenating every page", async () => {
+      const { client, fetchMock } = await authedClient();
+      fetchMock
+        .mockResolvedValueOnce(
+          jsonResponse({
+            data: [{ id: 1 }, { id: 2 }],
+            meta: { pagination: { total: 3, current: 1, lastPage: 2, limit: 2 } },
+          }),
+        )
+        .mockResolvedValueOnce(
+          jsonResponse({
+            data: [{ id: 3 }],
+            meta: { pagination: { total: 3, current: 2, lastPage: 2, limit: 2 } },
+          }),
+        );
+
+      const result = await client.getAll("/groups", { limit: 2 });
+
+      expect(result.data).toEqual([{ id: 1 }, { id: 2 }, { id: 3 }]);
+      expect(result.meta?.pagination).toMatchObject({ total: 3, current: 2, lastPage: 2 });
+      // calls 0-1 are the auth handshake (whoami + csrftoken); 2-3 are the two pages.
+      expect(String(fetchMock.mock.calls[2]?.[0])).toContain("page=1");
+      expect(String(fetchMock.mock.calls[2]?.[0])).toContain("limit=2");
+      expect(String(fetchMock.mock.calls[3]?.[0])).toContain("page=2");
+    });
+
+    it("stops after a single page when the response carries no pagination meta", async () => {
+      const { client, fetchMock } = await authedClient();
+      fetchMock.mockResolvedValueOnce(jsonResponse({ data: [{ id: 1 }] }));
+
+      const result = await client.getAll("/campuses");
+
+      expect(result.data).toEqual([{ id: 1 }]);
+      expect(fetchMock).toHaveBeenCalledTimes(3); // 2 auth + 1 page, no second page requested
+    });
+
+    it("defaults the per-page limit to 100", async () => {
+      const { client, fetchMock } = await authedClient();
+      fetchMock.mockResolvedValueOnce(
+        jsonResponse({ data: [], meta: { pagination: { total: 0, current: 1, lastPage: 1, limit: 100 } } }),
+      );
+
+      await client.getAll("/groups");
+
+      expect(String(fetchMock.mock.calls[2]?.[0])).toContain("limit=100");
+    });
+
+    it("stops when a page comes back empty even if lastPage claims more", async () => {
+      const { client, fetchMock } = await authedClient();
+      fetchMock.mockResolvedValueOnce(
+        jsonResponse({ data: [], meta: { pagination: { total: 3, current: 1, lastPage: 2, limit: 2 } } }),
+      );
+
+      const result = await client.getAll("/groups", { limit: 2 });
+
+      expect(result.data).toEqual([]);
+      expect(fetchMock).toHaveBeenCalledTimes(3); // does not spin forever on a degenerate response
+    });
+
+    it("propagates a CtApiError with status + body from a failing page", async () => {
+      const { client, fetchMock } = await authedClient();
+      fetchMock.mockResolvedValueOnce(
+        new Response(JSON.stringify({ errors: ["limit exceeds max of 100"] }), { status: 400 }),
+      );
+
+      await expect(client.getAll("/groups", { limit: 500 })).rejects.toMatchObject({
+        name: "CtApiError",
+        status: 400,
+        body: { errors: ["limit exceeds max of 100"] },
+      });
+    });
+  });
 });