Merge pull request #68 from CodeForPhilly/worktree-agent-aacd6f51bc5cf5c00

Port data-repo reconciliation into the API process (closes #66)

Author: themightychris

apps/api/src/app.ts

@@ -8,6 +8,11 @@
  *  4. trace-id plugin      → UUIDv7 traceId on every request
  *  5. setErrorHandler      → single error mapper for all throws
  *  6. store plugin         → decorates fastify.store from bootStores()
+ *  6a. reconcile plugin    → fetch + ff/rebase/escape-hatch against origin
+ *                            (between store and services so in-memory state
+ *                             is built from the post-reconciliation tree)
+ *  6b. push-daemon plugin  → starts gitsheets push daemon
+ *  6c. services plugin     → builds in-memory state + FTS
  *  7. rate-limit plugin    → in-memory counters keyed per-IP + per-account
  *  8. idempotency plugin   → in-memory map keyed by personId+key
  *  9. @fastify/swagger      → OpenAPI 3.1 doc generation
@@ -29,6 +34,7 @@ import { envJsonSchema, type Env } from './env.js';
 import { mapError } from './lib/errors.js';
 import traceIdPlugin from './plugins/trace-id.js';
 import storePlugin from './plugins/store.js';
+import reconcilePlugin from './plugins/reconcile.js';
 import pushDaemonPlugin from './plugins/push-daemon.js';
 import servicesPlugin from './plugins/services.js';
 import rateLimitPlugin from './plugins/rate-limit.js';
@@ -111,10 +117,16 @@ export async function buildApp(opts: BuildAppOptions = {}): Promise<FastifyInsta
   // ----- 6. Store (boots gitsheets + private-store) -----
   await fastify.register(storePlugin);
 
-  // ----- 6a. Push daemon (pushes public-store commits to CFP_DATA_REMOTE) -----
+  // ----- 6a. Reconcile (fetch + ff/rebase/escape-hatch against origin) -----
+  // Runs AFTER store (needs the repo handle) and BEFORE services (so the
+  // in-memory state is built from the post-reconciliation tree). Skipped
+  // when CFP_DATA_REMOTE is unset.
+  await fastify.register(reconcilePlugin);
+
+  // ----- 6b. Push daemon (pushes public-store commits to CFP_DATA_REMOTE) -----
   await fastify.register(pushDaemonPlugin);
 
-  // ----- 6b. Services (loads in-memory state + FTS, boots after store) -----
+  // ----- 6c. Services (loads in-memory state + FTS, boots after store) -----
   await fastify.register(servicesPlugin);
 
   // ----- 7. Rate limiting -----

apps/api/src/lib/data-repo-lock.ts

@@ -0,0 +1,54 @@
+/**
+ * Single-slot async lock for serializing data-repo operations that bypass
+ * `store.transact` — namely, the boot-time reconciliation (this plan) and
+ * the future hot-reload webhook (#65).
+ *
+ * Why not reuse gitsheets' internal `Mutex`? It serializes calls to
+ * `Repository.transact` but it is per-Repository-instance and isn't exposed
+ * on the public Repository surface — we'd have to reach through internals.
+ * A dedicated lock at the Fastify layer is the cleanest place to coordinate
+ * reconciliation against future webhook-driven transacts.
+ *
+ * At boot there's no contention; the lock is uncontended and overhead is
+ * a microtask. Once #65 lands, the webhook handler acquires this lock
+ * before fetching + rebuilding the in-memory state, and any concurrent
+ * write request that calls `store.transact` will wait inside gitsheets'
+ * internal mutex (which the webhook avoids holding while it does the
+ * external git fetch). Reconciliation and transacts therefore stay
+ * mutually exclusive as long as #65's handler acquires this lock for the
+ * duration of `reconcileDataRepo` AND defers any in-memory rebuild until
+ * after release.
+ */
+
+export type DataRepoLockRelease = () => void;
+export type DataRepoLock = () => Promise<DataRepoLockRelease>;
+
+/**
+ * Create a fresh single-slot lock. Multiple callers calling `acquire()`
+ * (the returned function) queue FIFO; only one holds the lock at a time.
+ *
+ * The returned release function is idempotent — calling it twice releases
+ * exactly once.
+ */
+export function createDataRepoLock(): DataRepoLock {
+  // Tail of the promise chain. Each acquire chains a new pending promise
+  // onto `tail`; the previous holder's release resolves the prior tail.
+  let tail: Promise<void> = Promise.resolve();
+
+  return async function acquire(): Promise<DataRepoLockRelease> {
+    let release!: () => void;
+    const next = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    const prior = tail;
+    tail = next;
+    await prior;
+
+    let released = false;
+    return (): void => {
+      if (released) return;
+      released = true;
+      release();
+    };
+  };
+}

apps/api/src/plugins/reconcile.ts

@@ -0,0 +1,163 @@
+/**
+ * Reconcile plugin.
+ *
+ * Replaces the data-repo reconciliation that used to live in
+ * `deploy/docker/entrypoint.sh`. Registered AFTER `storePlugin` (so the
+ * repository handle is available) and BEFORE `servicesPlugin` (so the
+ * in-memory state is built from the post-reconciliation tree).
+ *
+ * Behavior:
+ *   - When `CFP_DATA_REMOTE` is unset, reconciliation is skipped entirely
+ *     (typical for local dev against a sibling working tree with no remote).
+ *   - Otherwise: calls `reconcileDataRepo` for the configured branch and
+ *     logs the outcome at the appropriate level:
+ *       - 'conflict-escaped' → ERROR with the `conflictBranch` field, so
+ *         operators see a loud line in production logs.
+ *       - 'fetch-failed'     → WARN — non-fatal, the API still boots from
+ *         local state.
+ *       - everything else    → INFO.
+ *   - Any other thrown error (corrupt repo, missing branch, etc.) propagates
+ *     and crashes the boot. k8s will restart the pod and the entrypoint will
+ *     re-clone if needed.
+ *
+ * Decorates Fastify with:
+ *   - `dataRepoLock` — a single-slot async lock callers use to serialize
+ *     non-`store.transact` git operations (boot reconcile, future webhook).
+ *   - `reconcileDataRepo({ branch })` — a thin wrapper that acquires the
+ *     lock and invokes the state-machine function with the current
+ *     environment. Provided so the future hot-reload webhook (#65) has a
+ *     single call to make.
+ */
+import type { FastifyInstance } from 'fastify';
+import fp from 'fastify-plugin';
+
+import { createDataRepoLock, type DataRepoLock } from '../lib/data-repo-lock.js';
+import { reconcileDataRepo, type ReconcileResult } from '../store/reconcile.js';
+
+declare module 'fastify' {
+  interface FastifyInstance {
+    /**
+     * Acquire the data-repo lock. Returns a release function; release is
+     * idempotent. See `lib/data-repo-lock.ts` for the contract.
+     */
+    dataRepoLock: DataRepoLock;
+    /**
+     * Reconcile the local working tree against `CFP_DATA_REMOTE` for the
+     * given branch under the data-repo lock. Defaults to the configured
+     * `CFP_DATA_BRANCH`.
+     *
+     * Returns the outcome envelope. Throws on unrecoverable filesystem /
+     * git errors; soft failures (fetch blip, conflict-escape) return a
+     * non-throwing result.
+     */
+    reconcileDataRepo: (opts?: { branch?: string }) => Promise<ReconcileResult>;
+  }
+}
+
+async function reconcilePlugin(fastify: FastifyInstance): Promise<void> {
+  const lock = createDataRepoLock();
+  fastify.decorate('dataRepoLock', lock);
+
+  const repoPath = fastify.config.CFP_DATA_REPO_PATH;
+  const configuredBranch = fastify.config.CFP_DATA_BRANCH;
+  const remote = fastify.config.CFP_DATA_REMOTE;
+
+  // Expose a Fastify-bound wrapper so the future webhook handler (#65) has
+  // a single call to make. Always under the lock.
+  fastify.decorate(
+    'reconcileDataRepo',
+    async (opts?: { branch?: string }): Promise<ReconcileResult> => {
+      const branch = opts?.branch ?? configuredBranch;
+      if (!branch) {
+        throw new Error(
+          'reconcileDataRepo: no branch specified and CFP_DATA_BRANCH is unset',
+        );
+      }
+      const release = await lock();
+      try {
+        return await reconcileDataRepo({
+          repoPath,
+          branch,
+          logger: fastify.log,
+        });
+      } finally {
+        release();
+      }
+    },
+  );
+
+  // Boot-time reconcile: skipped when no remote is configured (dev).
+  if (!remote) {
+    fastify.log.info(
+      'data-repo reconciliation skipped: CFP_DATA_REMOTE unset (dev mode)',
+    );
+    return;
+  }
+
+  if (!configuredBranch) {
+    // Without a branch, we don't know what to reconcile against. Treat as
+    // a configuration error — entrypoint should set CFP_DATA_BRANCH
+    // alongside CFP_DATA_REMOTE.
+    throw new Error(
+      'data-repo reconciliation: CFP_DATA_REMOTE set but CFP_DATA_BRANCH unset; refusing to guess',
+    );
+  }
+
+  const release = await lock();
+  let result: ReconcileResult;
+  try {
+    result = await reconcileDataRepo({
+      repoPath,
+      branch: configuredBranch,
+      logger: fastify.log,
+    });
+  } finally {
+    release();
+  }
+
+  // Outcome-specific logging so operators get an at-a-glance line in prod.
+  switch (result.outcome) {
+    case 'conflict-escaped':
+      // LOUD: the operator MUST investigate the named branch.
+      fastify.log.error(
+        {
+          branch: configuredBranch,
+          conflictBranch: result.conflictBranch,
+          oldCommit: result.oldCommit,
+          newCommit: result.newCommit,
+          ahead: result.ahead,
+          behind: result.behind,
+        },
+        'data-repo reconciliation invoked conflict escape hatch',
+      );
+      break;
+    case 'fetch-failed':
+      fastify.log.warn(
+        { branch: configuredBranch, commit: result.oldCommit },
+        'data-repo reconciliation: fetch failed; continuing with local state',
+      );
+      break;
+    case 'in-sync':
+    case 'fast-forwarded':
+    case 'pushed-ahead':
+    case 'rebased':
+      fastify.log.info(
+        {
+          branch: configuredBranch,
+          outcome: result.outcome,
+          oldCommit: result.oldCommit,
+          newCommit: result.newCommit,
+          ahead: result.ahead,
+          behind: result.behind,
+        },
+        'data-repo reconciled',
+      );
+      break;
+  }
+}
+
+export default fp(reconcilePlugin, {
+  name: 'reconcile',
+  fastify: '5.x',
+  dependencies: ['store'],
+});