From f5a4d2e24586425f054d35071c76b47078e3cd32 Mon Sep 17 00:00:00 2001 From: yai-dev Date: Mon, 13 Apr 2026 18:12:44 +0800 Subject: [PATCH 1/2] feat: storage abstraction, PostgreSQL backend, and memo mirror contract (#69) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Core storage contract changes: - `AgentrailSessionStore` gains optional memo document and tool-result artifact methods (read/write/append/list); required for non-filesystem backends to enable sandbox /workspace/memo/** access - `AgentrailSessionStore` no longer exposes `getSessionDir`; removed from all host call sites and developer docs - `MemoryIndex` paths use canonical /workspace/memo/... form only SandboxManager memo mirror: - New `SandboxMemoProvider` interface with bidirectional read/write methods - At sandbox creation: snapshots memo docs + tool-result artifacts into a temp dir; bind-mounts read-only so Bash cannot bypass Write/Edit tools - New `writeMemoBack(sessionId, containerPath, content)` — Write/Edit tools call this after every successful memo-path write to keep store in sync - New `refreshMemoMirror` / `refreshUserMemoMirrorForAllSessions` — host calls these after compaction artifact writes or USER.md consolidation so live containers see updated content immediately - `SandboxEntry` stores tenantId/userId/sessionId for write-back routing UserMemoryConsolidationService: - New optional `mirrorRefresher: UserMemoMirrorRefresher` constructor param; calls `refreshUserMemoMirrorForAllSessions` after every USER.md write memoryContext capability: - `MemorySessionContext` gains `sessionRef` field - `MemoryContextBuilders` gains `writeToolResultArtifact` builder; wired through to `DefaultCapabilityContextOptions.writeToolResultArtifact` so compaction artifacts are actually persisted (previous `sessionDir` path was never populated and silently dropped all artifacts) - `compactMessages` ctx type updated: `sessionDir` removed, replaced by `writeToolResultArtifact` callback OrchestrationPersistence: - Adds `loadAgentHistory` / `writeAgentHistory` for sub-agent message history across turn boundaries - Worker process supports dynamic postgres persistence init via `storageConfig` in `WorkerInitMessage` InspectorDataSource: - New `InspectorDataSource` interface + `createFilesystemInspectorDataSource` - `createInspectorRoute` accepts `InspectorDataSource` (string path deprecated) - `createAgentApp` `inspector` option now takes `true | InspectorDataSource` New package: @agentrail/storage-postgres: - `PostgresSessionStore` — full store including memo docs and artifacts - `PostgresSessionTraceStore` - `PostgresOrchestrationPersistence` — includes agent history - `PostgresInspectorDataSource` - `buildSchemaDDL(schema?)` for idempotent table creation - `createSqlClient(options)` connection pool helper - Full test suite via @testcontainers/postgresql playground-server example: - Updated `compactMessages` to use `writeToolResultArtifact` + mirror refresh - Stream route uses cached trace store (one per session, not one per event) - Inspector route switched to `createFilesystemInspectorDataSource` Docs: - New guide: docs/guides/build-a-storage-backend.md covering all 6 contracts and mirror refresh responsibilities - session-store.md, configure-sessions.md, inspector-route.md updated for new API; getSessionDir removed from public contract docs - playground-server.md examples updated to new patterns - docs/tools/bash.md + sandboxed-bash description: memo paths are read-only Signed-off-by: yai-dev --- .changeset/storage-abstraction-postgres.md | 57 ++ docs/concepts/sessions.md | 5 +- docs/examples/playground-server.md | 58 +- docs/guides/build-a-storage-backend.md | 491 +++++++++++++ docs/guides/configure-sessions.md | 73 +- docs/public/llms.txt | 26 + docs/reference/create-agent-app.md | 78 ++- docs/reference/inspector-route.md | 78 ++- docs/reference/session-store.md | 196 ++++-- docs/tools/bash.md | 3 +- examples/README.md | 16 + .../playground-server/src/context/index.ts | 12 +- examples/playground-server/src/main.ts | 7 +- .../src/profiles/default-profile.ts | 13 +- .../playground-server/src/routes/stream.ts | 25 +- packages/app/src/advanced.ts | 2 + packages/app/src/app/create-agent-app.ts | 240 +++++-- .../src/host/defaults/capability-messages.ts | 29 +- .../app/src/host/defaults/shared-types.ts | 6 +- .../app/src/host/orchestration-registry.ts | 29 +- packages/app/src/index.ts | 9 + packages/app/src/inspector/data-source.ts | 242 +++++++ packages/app/src/inspector/index.ts | 276 ++------ .../user-memory-consolidation-service.ts | 108 ++- packages/app/src/session/compaction-logic.ts | 123 ++++ packages/app/src/session/compaction.ts | 31 +- packages/app/src/session/memo-fs.ts | 101 +++ packages/app/src/session/session-manager.ts | 191 +++-- .../app/src/session/user-session-lister.ts | 20 + packages/app/test/reactive-compaction.test.ts | 17 +- packages/capabilities/src/index.ts | 4 + packages/capabilities/src/memory/context.ts | 9 +- packages/capabilities/src/memory/index.ts | 51 +- packages/capabilities/src/memory/messages.ts | 36 +- packages/capabilities/src/memory/types.ts | 12 +- .../src/orchestration/persistence.ts | 14 + .../orchestration/worker/subagent-worker.ts | 80 ++- .../orchestration/worker/worker-messages.ts | 21 + packages/capabilities/src/sandbox/index.ts | 1 + .../src/sandbox/sandbox-manager.ts | 376 +++++++++- .../src/sandbox/tools/sandboxed-bash.ts | 4 +- .../src/sandbox/tools/sandboxed-edit.ts | 1 + .../src/sandbox/tools/sandboxed-write.ts | 1 + .../test/memory-context-compaction.test.ts | 35 +- packages/core/src/index.ts | 2 + packages/core/src/session/contracts.ts | 76 ++ packages/core/src/session/types.ts | 16 +- packages/storage-postgres/package.json | 49 ++ packages/storage-postgres/src/client.ts | 54 ++ packages/storage-postgres/src/index.ts | 14 + .../src/inspector-data-source.ts | 146 ++++ .../src/orchestration-persistence.ts | 167 +++++ packages/storage-postgres/src/schema.ts | 185 +++++ .../storage-postgres/src/session-store.ts | 337 +++++++++ packages/storage-postgres/src/trace-store.ts | 60 ++ packages/storage-postgres/test/helpers.ts | 70 ++ .../test/inspector-data-source.test.ts | 129 ++++ .../test/orchestration-persistence.test.ts | 157 +++++ .../test/session-store.test.ts | 149 ++++ .../storage-postgres/test/trace-store.test.ts | 83 +++ packages/storage-postgres/tsconfig.json | 12 + packages/storage-postgres/tsconfig.test.json | 13 + packages/storage-postgres/vitest.config.ts | 25 + pnpm-lock.yaml | 655 +++++++++++++++++- 64 files changed, 5037 insertions(+), 569 deletions(-) create mode 100644 .changeset/storage-abstraction-postgres.md create mode 100644 docs/guides/build-a-storage-backend.md create mode 100644 examples/README.md create mode 100644 packages/app/src/inspector/data-source.ts create mode 100644 packages/app/src/session/compaction-logic.ts create mode 100644 packages/app/src/session/memo-fs.ts create mode 100644 packages/app/src/session/user-session-lister.ts create mode 100644 packages/storage-postgres/package.json create mode 100644 packages/storage-postgres/src/client.ts create mode 100644 packages/storage-postgres/src/index.ts create mode 100644 packages/storage-postgres/src/inspector-data-source.ts create mode 100644 packages/storage-postgres/src/orchestration-persistence.ts create mode 100644 packages/storage-postgres/src/schema.ts create mode 100644 packages/storage-postgres/src/session-store.ts create mode 100644 packages/storage-postgres/src/trace-store.ts create mode 100644 packages/storage-postgres/test/helpers.ts create mode 100644 packages/storage-postgres/test/inspector-data-source.test.ts create mode 100644 packages/storage-postgres/test/orchestration-persistence.test.ts create mode 100644 packages/storage-postgres/test/session-store.test.ts create mode 100644 packages/storage-postgres/test/trace-store.test.ts create mode 100644 packages/storage-postgres/tsconfig.json create mode 100644 packages/storage-postgres/tsconfig.test.json create mode 100644 packages/storage-postgres/vitest.config.ts diff --git a/.changeset/storage-abstraction-postgres.md b/.changeset/storage-abstraction-postgres.md new file mode 100644 index 0000000..0dc34fd --- /dev/null +++ b/.changeset/storage-abstraction-postgres.md @@ -0,0 +1,57 @@ +--- +"@agentrail/core": minor +"@agentrail/capabilities": minor +"@agentrail/app": minor +"@agentrail/storage-postgres": minor +--- + +**Storage Abstraction + Memo FS + PostgreSQL reference implementation (#69)** + +This release completes the Pre-GA storage contract consolidation and ships the `@agentrail/storage-postgres` reference package. + +### `@agentrail/core` — breaking shape changes + new optional methods + +- `MemoryIndex` no longer contains `sessionDir` or `userDir` host-absolute paths. The `entries[].path` field now stores the canonical `/workspace/memo/…` path directly. +- `AgentrailSessionStore` gains five new **optional** methods for memo-document and tool-result-artifact persistence: + - `readMemoryDocument(tenantId, ownerId, scope, name)` + - `writeMemoryDocument(tenantId, ownerId, scope, name, content)` + - `appendMemoryDocument(tenantId, ownerId, scope, name, content)` + - `readToolResultArtifact(sessionRef, toolCallId)` + - `writeToolResultArtifact(sessionRef, toolCallId, content)` +- New types exported: `MemoDocumentScope`, `MemoDocumentName` + +### `@agentrail/capabilities` — new exports + breaking signature + +- `OrchestrationPersistence` gains two new required methods: `loadAgentHistory(agentId)` and `writeAgentHistory(agentId, history)`. +- `DefaultCapabilityContextOptions.compactMessages` signature updated: `sessionDir` replaced by `writeToolResultArtifact` callback injection. +- New public exports: `OrchestrationMailboxEvent`, `OrchestrationSnapshot`, `recoverOrchestrationState`. +- `WorkerInitMessage` gains an optional `storageConfig` field for sub-agent worker initialization. + +### `@agentrail/app` — breaking API surface changes + +- `CreateAgentAppOptions` gains: `traceStoreFactory`, `createOrchestrationPersistence`, `createWorkerStorageConfig`, and `inspector` (now accepts `true | InspectorDataSource`). +- `createInspectorRoute` now accepts `InspectorDataSource | string` (string is deprecated backward-compat path; direct `dataDir` strings will be removed in a future release). +- `UserMemoryConsolidationService` constructor signature changed from `(sessionManager, dataDir, config)` to `(store: AgentrailSessionStore, sessionLister: UserSessionLister, dataDir, config)`. +- New `UserSessionLister` interface exported. +- New `InspectorDataSource`, `InspectorSessionItem`, `SessionTraceStore` types exported. +- New `createFilesystemInspectorDataSource(dataDir)` function exported. +- `compactToolResults` now prefers `writeToolResultArtifact` from the session store when available. +- Pure compaction helpers exported: `computeCompactionSplit`, `buildCompactionMessage`, `buildCompactionNotesEntry`. + +### `@agentrail/storage-postgres` — new package + +PostgreSQL reference implementation for all Agentrail storage contracts: + +- `PostgresSessionStore` — full `AgentrailSessionStore` including memo documents and tool-result artifacts. +- `PostgresSessionTraceStore` + `createPostgresSessionTraceStore` factory. +- `PostgresOrchestrationPersistence` + `createPostgresOrchestrationPersistence` factory (includes agent history). +- `PostgresInspectorDataSource` — reads session list, messages, trace envelopes, and orchestration state from PostgreSQL without any filesystem dependency. +- `buildSchemaDDL(schema?)` — returns idempotent DDL to create all required tables. +- `createSqlClient(options)` — creates a `postgres` (postgres.js) connection pool. + +**Upgrade notes:** + +1. If you implement `AgentrailSessionStore`, the new optional methods do not need to be implemented immediately; the runtime falls back gracefully. However, implementing them is required to use the postgres backend. +2. If you implement `OrchestrationPersistence`, you must add `loadAgentHistory` and `writeAgentHistory`. +3. Replace `createInspectorRoute(dataDir)` with `createInspectorRoute(createFilesystemInspectorDataSource(dataDir))` when providing a custom `InspectorDataSource`. +4. `UserMemoryConsolidationService` callers must provide a `UserSessionLister` (the default `SessionManager` now implements this interface). diff --git a/docs/concepts/sessions.md b/docs/concepts/sessions.md index c822de0..9b962ad 100644 --- a/docs/concepts/sessions.md +++ b/docs/concepts/sessions.md @@ -32,18 +32,19 @@ getOrCreate → existing session resumed The host layer depends on the `AgentrailSessionStore` contract rather than any concrete storage implementation. This means you can swap out the storage backend without changing your host code. -The contract includes: +The required contract includes: | Method | Called when | | ------------------------ | --------------------------------------------------------------------------- | | `getOrCreate` | Every request — creates or resumes the session | -| `getSessionDir` | Before agent construction — returns the session data path | | `loadMessagesWithBudget` | Every request — loads history trimmed to the token budget | | `loadAllMessages` | During compaction — loads full history to assess whether to compact | | `appendMessages` | After agent completes — persists the new turn's messages | | `recordTurn` | After agent completes — persists token usage | | `compactIfNeeded` | During each request — summarizes old history if token threshold is exceeded | +Additional optional methods (`readMemoryDocument`, `writeMemoryDocument`, `readToolResultArtifact`, `writeToolResultArtifact`, `listToolResultArtifactIds`) unlock memo tools and full `/workspace/memo/**` sandbox access. See [Session Store Reference](../reference/session-store.md) for the complete interface. + ## Default Implementation: `SessionManager` The default session store is `SessionManager` from `@agentrail/app`. It uses the local filesystem to persist sessions, with one directory per session: diff --git a/docs/examples/playground-server.md b/docs/examples/playground-server.md index 7a791bc..078ee42 100644 --- a/docs/examples/playground-server.md +++ b/docs/examples/playground-server.md @@ -31,11 +31,28 @@ The two host entry points are mounted in `routes/chat.ts` and `routes/stream.ts` // examples/playground-server/src/routes/stream.ts (simplified) import { createStreamRoute } from "@agentrail/app/advanced"; import { createFileSystemSessionTraceStore } from "@agentrail/app"; +import type { WorkflowTraceEventEnvelope } from "@agentrail/app"; +import type { SessionRef } from "@agentrail/core"; import { resolvePlaygroundProfile } from "../profiles/default-profile.js"; import { sessionManager, sandboxManager, orchestrationRegistry } from "../context/index.js"; import { plugins } from "../plugins/index.js"; import { summarize } from "../agents/summarizer.js"; +// Cache one trace store per session so the underlying file is opened once per +// process lifetime, not once per event. +const traceStoreCache = new Map< + SessionRef, + ReturnType> +>(); +function getTraceStore(sessionRef: SessionRef) { + let store = traceStoreCache.get(sessionRef); + if (!store) { + store = createFileSystemSessionTraceStore(dataDir, sessionRef); + traceStoreCache.set(sessionRef, store); + } + return store; +} + export const streamRoute = createStreamRoute({ dataDir, defaultAgentId: "default", @@ -58,14 +75,34 @@ export const streamRoute = createStreamRoute({ getOrchestrationManager: ({ tenantId, userId, sessionId, sessionRef }) => orchestrationRegistry.getManager({ tenantId, userId, sessionId, sessionRef }), onTraceEvent: (ctx, envelope) => { - const traceStore = createFileSystemSessionTraceStore(dataDir, ctx.sessionRef); - void traceStore.appendEnvelope(envelope); + void getTraceStore(ctx.sessionRef).appendEnvelope(envelope); }, }); ``` Both chat and stream routes share the same session store, profile resolver, and plugin list — that reuse is the main design goal. +### Inspector Route + +The Inspector API is mounted as a sub-app. Pass an `InspectorDataSource` to `createInspectorRoute` — for filesystem-backed setups use `createFilesystemInspectorDataSource`; for database backends, pass a `PostgresInspectorDataSource` or your own implementation: + +```ts +// examples/playground-server/src/main.ts (simplified) +import { createInspectorRoute, createFilesystemInspectorDataSource } from "@agentrail/app/advanced"; + +// Filesystem-backed (default playground setup) +app.route( + "/__inspector", + createInspectorRoute(createFilesystemInspectorDataSource(config.dataDir)), +); + +// PostgreSQL backend — swap the data source, everything else stays the same +// import { PostgresInspectorDataSource } from "@agentrail/storage-postgres"; +// app.route("/__inspector", createInspectorRoute(new PostgresInspectorDataSource(sql))); +``` + +`createInspectorRoute` is backend-agnostic: it only calls methods on the `InspectorDataSource` interface, so switching from filesystem to PostgreSQL is a one-line change. + ### Profile Definition The default profile lives in `profiles/default-profile.ts`. It shows the recommended `defineProfile` shape with capability descriptors: @@ -133,10 +170,21 @@ export const defaultProfile = defineProfile({ }, listSkills: () => skillManager.listSkills(), listWorkspaceSnapshot: (ctx) => sandboxManager.listWorkspace(ctx.sessionId), + // Persists compacted tool results to the session store and refreshes + // the live sandbox mirror so the agent can read the file immediately. + writeToolResultArtifact: async (ctx, toolCallId, content) => { + const sessionRef = `${ctx.tenantId}:${ctx.sessionId}`; + await Promise.all([ + sessionManager.writeToolResultArtifact?.(sessionRef, toolCallId, content), + sandboxManager.refreshMemoMirror( + ctx.sessionId, + `/workspace/memo/session/tool-results/${toolCallId}.txt`, + content, + ), + ]); + }, compactMessages: (msgs, ctx) => - compactToolResults(msgs, { - sessionDir: ctx?.sessionDir, - }), + compactToolResults(msgs, { writeToolResultArtifact: ctx?.writeToolResultArtifact }), delegateSkillsToSubAgent: config.skillDelegateToSubAgent, }, { cacheTtlMs: 5_000 }, diff --git a/docs/guides/build-a-storage-backend.md b/docs/guides/build-a-storage-backend.md new file mode 100644 index 0000000..26fffbc --- /dev/null +++ b/docs/guides/build-a-storage-backend.md @@ -0,0 +1,491 @@ +# Build a Storage Backend + +This guide explains every interface a custom storage backend must implement to fully replace the default filesystem layer in an Agentrail host. + +## When to read this + +Read this guide when you need to run Agentrail with a storage backend other than the local filesystem — for example PostgreSQL, MySQL, SQLite, Redis, or a managed object store. The built-in `@agentrail/storage-postgres` package implements all of the contracts below and serves as the reference implementation. + +## Prerequisites + +- [Configure Sessions](configure-sessions.md) +- [Session Store Reference](../reference/session-store.md) +- [Inspector Route Reference](../reference/inspector-route.md) + +--- + +## Contract overview + +A complete custom backend touches six distinct contracts, plus mirror refresh responsibilities that apply when a sandbox is active. Not all are required for every deployment; the table below shows which features each one unlocks: + +| Contract | Required for | Package | +| -------------------------- | ----------------------------------------------- | ------------------------------------ | +| `AgentrailSessionStore` | All requests — messages, compaction, turns | `@agentrail/core` / `@agentrail/app` | +| `UserSessionLister` | User-memory consolidation background service | `@agentrail/app` | +| `SessionTraceStore` | Workflow trace persistence (Inspector timeline) | `@agentrail/app` | +| `OrchestrationPersistence` | Multi-agent orchestration state | `@agentrail/capabilities` | +| `InspectorDataSource` | Agentrail Inspector read API | `@agentrail/app` | +| `SandboxMemoProvider` | `/workspace/memo/**` inside containers | `@agentrail/capabilities` | + +The sections below cover each contract in turn, then show how to wire them into `createAgentApp`. + +--- + +## 1. `AgentrailSessionStore` + +The core per-request contract. Seven methods are required; the optional memo and artifact methods unlock memory tools and full sandbox access. + +### Required methods + +```ts +import type { AgentrailSessionStore } from "@agentrail/app"; +import type { Message, SessionRef, Usage } from "@agentrail/core"; + +export class MySessionStore implements AgentrailSessionStore { + async getOrCreate(tenantId, userId, agentId, sessionId?) { + const id = sessionId ?? crypto.randomUUID(); + await db.sessions.upsert({ id, tenantId, userId, agentId }); + return { sessionId: id }; + } + + async loadMessages(tenantId, sessionId, limit?) { + return db.messages.findMany({ sessionId, limit }); + } + + async loadMessagesWithBudget(tenantId, sessionId, tokenBudget = 100_000) { + const all = await db.messages.findAll({ sessionId, orderBy: "asc" }); + // trim from the oldest end until total fits in budget + return trimToTokenBudget(all, tokenBudget); + } + + async loadAllMessages(tenantId, sessionId) { + return db.messages.findAll({ sessionId }); + } + + async appendMessages(tenantId, sessionId, messages: Message[]) { + await db.messages.insertMany(messages.map((m) => ({ ...m, sessionId }))); + } + + async recordTurn(tenantId, sessionId, usage: Usage) { + await db.usage.insert({ sessionId, ...usage }); + } + + async compactIfNeeded(tenantId, sessionId, summarizeFn, options?) { + const messages = await this.loadAllMessages(tenantId, sessionId); + const triggerTokens = options?.triggerTokens ?? 80_000; + if (estimateTokens(messages) < triggerTokens) return false; + + const compactFraction = options?.compactFraction ?? 0.5; + const cutoff = Math.floor(messages.length * compactFraction); + const summary = await summarizeFn(messages.slice(0, cutoff)); + + const compacted: Message[] = [ + { role: "user", content: `[Conversation summary]: ${summary}` }, + ...messages.slice(cutoff), + ]; + await db.messages.replaceAll(sessionId, compacted); + return true; + } +} +``` + +### Optional memo methods + +Implement these to enable in-context memo tools (`write_notes`, `write_todo`) and full `/workspace/memo/**` access inside containers. + +```ts +async readMemoryDocument(tenantId, ownerId, scope, name) { + return db.memoDocuments.findOne({ tenantId, ownerId, scope, name }) ?? null; +} + +async writeMemoryDocument(tenantId, ownerId, scope, name, content) { + await db.memoDocuments.upsert({ tenantId, ownerId, scope, name, content }); +} + +async appendMemoryDocument(tenantId, ownerId, scope, name, content) { + const existing = (await this.readMemoryDocument(tenantId, ownerId, scope, name)) ?? ""; + await this.writeMemoryDocument(tenantId, ownerId, scope, name, existing + content); +} +``` + +### Optional tool-result artifact methods + +Implement these to persist compacted tool-result artifacts. Without them, compaction still works but large tool outputs cannot be retrieved from inside the sandbox later. + +```ts +async readToolResultArtifact(sessionRef, toolCallId) { + return db.toolResultArtifacts.findOne({ sessionRef, toolCallId }) ?? null; +} + +async writeToolResultArtifact(sessionRef, toolCallId, content) { + await db.toolResultArtifacts.upsert({ sessionRef, toolCallId, content }); +} + +async listToolResultArtifactIds(sessionRef) { + return db.toolResultArtifacts.findIds({ sessionRef }); +} +``` + +`listToolResultArtifactIds` is used by `SandboxManager` at sandbox-creation time to pre-populate `/workspace/memo/session/tool-results/` inside the container. Without it, that directory starts empty even if artifacts exist in the store. + +--- + +## 2. `UserSessionLister` + +The user-memory consolidation background service uses this interface to scan sessions when deciding whether to rebuild a user's `USER.md` profile. `SessionManager` implements it automatically; for custom backends, implement it alongside (or as part of) your session store: + +```ts +import type { UserSessionLister } from "@agentrail/app"; +import type { SessionMeta } from "@agentrail/core"; + +export class MySessionStore implements AgentrailSessionStore, UserSessionLister { + // ... required session store methods above ... + + async listSessionsByUser(tenantId, userId): Promise { + return db.sessions.findMany({ + tenantId, + userId, + select: ["sessionId", "updatedAt"], + }); + } +} +``` + +Pass the same instance as both `sessionStore` and the second argument to `UserMemoryConsolidationService`: + +```ts +import { UserMemoryConsolidationService } from "@agentrail/app"; + +const store = new MySessionStore(); + +const consolidationService = new UserMemoryConsolidationService( + store, // AgentrailSessionStore + store, // UserSessionLister + dataDir, + config, + sandboxManager, // optional — propagates USER.md updates to live mirrors +); +``` + +--- + +## 3. `SessionTraceStore` + +The trace store persists workflow events (turn start/end, tool calls, errors) for later replay in the Agentrail Inspector timeline. The interface is minimal: + +```ts +export interface SessionTraceStore> { + appendEnvelope(envelope: TEnvelope): Promise; + loadEnvelopes(): Promise; +} +``` + +Implement a factory and pass it to `createAgentApp`: + +```ts +import type { SessionRef } from "@agentrail/core"; +import type { SessionTraceStore, WorkflowTraceEventEnvelope } from "@agentrail/app"; + +function createDbSessionTraceStore( + sessionRef: SessionRef, +): SessionTraceStore { + return { + async appendEnvelope(envelope) { + await db.traceEvents.insert({ sessionRef, ...envelope }); + }, + async loadEnvelopes() { + return db.traceEvents.findAll({ sessionRef, orderBy: "sequence" }); + }, + }; +} +``` + +The factory is called at most once per session (cached in `createAgentApp`), so it is safe to open a database connection or prepare a statement inside the factory. + +--- + +## 4. `OrchestrationPersistence` + +Multi-agent orchestration requires a session-scoped persistence layer to checkpoint agent state, buffer mailbox events, and recover after crashes. + +```ts +import type { OrchestrationPersistence } from "@agentrail/capabilities"; + +function createDbOrchestrationPersistence(sessionRef: SessionRef): OrchestrationPersistence { + return { + async appendEvent(event) { + await db.orchEvents.insert({ sessionRef, ...event }); + }, + async loadEvents() { + return db.orchEvents.findAll({ sessionRef, orderBy: "sequence" }); + }, + async loadSnapshot() { + return db.orchSnapshots.findLatest({ sessionRef }) ?? null; + }, + async writeCheckpoint(snapshot) { + await db.orchSnapshots.upsert({ sessionRef, snapshot }); + }, + async recoverState() { + const snapshot = await this.loadSnapshot(); + const events = await this.loadEvents(); + return recoverOrchestrationState(snapshot, events); // framework helper + }, + async appendMailboxEvent(agentId, event) { + await db.mailboxEvents.insert({ sessionRef, agentId, ...event }); + }, + async loadMailboxEvents(agentId) { + return db.mailboxEvents.findAll({ sessionRef, agentId }); + }, + async loadMailboxState(agentId) { + return db.mailboxStates.findOne({ sessionRef, agentId }) ?? {}; + }, + async writeMailboxState(agentId, state) { + await db.mailboxStates.upsert({ sessionRef, agentId, state }); + }, + async loadAgentHistory(agentId) { + return db.agentHistory.findOne({ sessionRef, agentId }) ?? []; + }, + async writeAgentHistory(agentId, history) { + await db.agentHistory.upsert({ sessionRef, agentId, history }); + }, + }; +} +``` + +Pass the factory to `createAgentApp`: + +```ts +const app = createAgentApp({ + sessionStore: store, + createOrchestrationPersistence: (sessionRef) => createDbOrchestrationPersistence(sessionRef), + profiles: [defaultProfile], +}); +``` + +--- + +## 5. `InspectorDataSource` + +`createInspectorRoute` reads session data through this interface. It is read-only and does not affect the request pipeline. + +```ts +import type { InspectorDataSource } from "@agentrail/app"; + +export class DbInspectorDataSource implements InspectorDataSource { + async listSessions() { + return db.sessions.findAll({ + select: ["tenantId", "sessionId", "userId", "updatedAt", "turns", "tokens"], + }); + } + + async loadMessages(tenantId, sessionId) { + return db.messages.findAll({ tenantId, sessionId, orderBy: "sequence" }); + } + + async loadTraceEnvelopes(tenantId, sessionId) { + const sessionRef = `${tenantId}:${sessionId}`; + return db.traceEvents.findAll({ sessionRef, orderBy: "sequence" }); + } + + async loadOrchestrationState(tenantId, sessionId) { + const sessionRef = `${tenantId}:${sessionId}`; + return db.orchSnapshots.findLatest({ sessionRef }) ?? null; + } + + async loadOrchestrationEvents(tenantId, sessionId) { + const sessionRef = `${tenantId}:${sessionId}`; + return db.orchEvents.findAll({ sessionRef, orderBy: "sequence" }); + } +} +``` + +Pass it to `createAgentApp`: + +```ts +const app = createAgentApp({ + inspector: new DbInspectorDataSource(), + profiles: [defaultProfile], +}); +``` + +--- + +## 6. `SandboxMemoProvider` (SandboxManager option) + +When a sandbox container is active, the agent reads `/workspace/memo/**` paths from the host filesystem via a bind-mount. For non-filesystem backends, `SandboxManager` needs a `memoProvider` to: + +1. **Snapshot** memo documents and tool-result artifacts into a temporary host directory at sandbox-creation time, then bind-mount that directory read-only into the container. +2. **Write back** agent edits (via the `Write` / `Edit` tools) to the backing store. +3. **Refresh** specific mirror files whenever the host updates the store outside the agent's tool path (compaction artifacts, USER.md consolidation). + +Any object that satisfies `SandboxMemoProvider` can be passed. Your session store likely already implements all of these methods: + +```ts +import { SandboxManager } from "@agentrail/capabilities"; + +const store = new MySessionStore(); + +const sandboxManager = new SandboxManager(process.env.AGENTRAIL_DATA_DIR!, { memoProvider: store }); +``` + +`SandboxMemoProvider` methods (all optional except `readMemoryDocument`): + +| Method | Direction | When called | +| --------------------------- | --------- | ---------------------------------------------------------- | +| `readMemoryDocument` | Read | Sandbox creation — snapshot NOTES.md, TODO.md, USER.md | +| `writeMemoryDocument` | Write | Agent uses `Write`/`Edit` on a memo path | +| `appendMemoryDocument` | Write | In-context memo tools (`write_notes`, `write_todo`) | +| `readToolResultArtifact` | Read | Sandbox creation (paired with `listToolResultArtifactIds`) | +| `writeToolResultArtifact` | Write | Agent uses `Write`/`Edit` on a tool-results path | +| `listToolResultArtifactIds` | Read | Sandbox creation — pre-populate tool-results directory | + +--- + +## 7. Mirror refresh responsibilities + +The `/workspace/memo/**` mount is **read-only** inside the container. The host can still update files on the host-side of the bind-mount, and the running container will see the change immediately. This is how compaction and memory consolidation deliver fresh content to a live sandbox. + +Two host-side write paths require a manual mirror refresh call: + +### 7a. Compacted tool-result artifacts (compaction) + +When `compactToolResults` writes a tool-result artifact, it calls your `writeToolResultArtifact` callback. Immediately after writing to the store, also write to the live sandbox mirror so the container can read the file without waiting for a restart. + +Wire this in your profile's `memoryContext` builder: + +```ts +import { memoryContext } from "@agentrail/capabilities"; +import { compactToolResults } from "@agentrail/app"; + +memoryContext({ + buildMemoryIndex: (ctx) => store.buildMemoryIndex(ctx.tenantId, ctx.userId, ctx.sessionId), + + writeToolResultArtifact: async (ctx, toolCallId, content) => { + await Promise.all([ + // 1. Persist to backing store + store.writeToolResultArtifact?.(ctx.sessionRef, toolCallId, content), + // 2. Refresh the live sandbox mirror + sandboxManager.refreshMemoMirror( + ctx.sessionId, + `/workspace/memo/session/tool-results/${toolCallId}.txt`, + content, + ), + ]); + }, + + compactMessages: (msgs, ctx) => + compactToolResults(msgs, { writeToolResultArtifact: ctx?.writeToolResultArtifact }), +}); +``` + +`sandboxManager.refreshMemoMirror` is a no-op when the session has no active sandbox or when using a filesystem backend (where the bind-mount IS the real path). + +### 7b. USER.md consolidation + +`UserMemoryConsolidationService` rewrites `USER.md` as a background task. Pass `sandboxManager` as the fifth constructor argument; the service automatically calls `sandboxManager.refreshUserMemoMirrorForAllSessions(...)` after every write: + +```ts +const consolidationService = new UserMemoryConsolidationService( + store, // AgentrailSessionStore + store, // UserSessionLister + dataDir, + memoryConfig, + sandboxManager, // propagates USER.md updates to all active session mirrors +); +``` + +--- + +## 8. Wiring everything in `createAgentApp` + +Once all pieces are implemented, pass them to `createAgentApp`: + +```ts +import { createAgentApp } from "@agentrail/app"; +import { SandboxManager } from "@agentrail/capabilities"; + +const store = new MySessionStore(); +const sandboxManager = new SandboxManager(process.env.AGENTRAIL_DATA_DIR!, { memoProvider: store }); + +const app = createAgentApp({ + sessionStore: store, + + sandboxManager, + + traceStoreFactory: (sessionRef) => createDbSessionTraceStore(sessionRef), + + createOrchestrationPersistence: (sessionRef) => createDbOrchestrationPersistence(sessionRef), + + inspector: new DbInspectorDataSource(), + + profiles: [defaultProfile], + summarize, + compaction: { + triggerTokens: 80_000, + minMessages: 20, + }, +}); +``` + +Start the consolidation service separately (it is not managed by `createAgentApp`): + +```ts +const consolidationService = new UserMemoryConsolidationService( + store, + store, + process.env.AGENTRAIL_DATA_DIR!, + memoryConfig, + sandboxManager, +); +consolidationService.start(); +``` + +--- + +## 9. Atomicity and concurrency notes + +The framework does not enforce transactions across contracts. If your backend requires consistency guarantees (e.g. atomically updating messages and usage), wrap them in a database transaction inside `appendMessages` + `recordTurn`. The call order is always `appendMessages` then `recordTurn` within a single turn. + +Trace and orchestration persistence calls are fire-and-forget from the framework's perspective — write failures are logged but never propagated to the agent. + +--- + +## 10. Reference implementation + +`@agentrail/storage-postgres` implements all seven contracts for PostgreSQL and can be used directly or studied as a reference: + +```ts +import { + PostgresSessionStore, + PostgresSessionTraceStore, + PostgresOrchestrationPersistence, + PostgresInspectorDataSource, + createSqlClient, +} from "@agentrail/storage-postgres"; + +const sql = createSqlClient({ connectionString: process.env.DATABASE_URL! }); + +const store = new PostgresSessionStore(sql); +const sandboxManager = new SandboxManager(dataDir, { memoProvider: store }); + +const app = createAgentApp({ + sessionStore: store, + sandboxManager, + traceStoreFactory: (sessionRef) => new PostgresSessionTraceStore(sql, sessionRef), + createOrchestrationPersistence: (sessionRef) => + new PostgresOrchestrationPersistence(sql, sessionRef), + inspector: new PostgresInspectorDataSource(sql), + profiles: [defaultProfile], +}); +``` + +`PostgresSessionStore` implements `AgentrailSessionStore`, `UserSessionLister`, and `SandboxMemoProvider` in a single class, so passing the same instance to multiple slots is correct and intended. + +--- + +## Related + +- [Configure Sessions](configure-sessions.md) +- [Session Store Reference](../reference/session-store.md) +- [Inspector Route Reference](../reference/inspector-route.md) +- [Deployment Guide](deployment.md) diff --git a/docs/guides/configure-sessions.md b/docs/guides/configure-sessions.md index ea55e15..d147b2a 100644 --- a/docs/guides/configure-sessions.md +++ b/docs/guides/configure-sessions.md @@ -106,9 +106,6 @@ const sessions = await sessionManager.listSessions(tenantId, userId); // Build a memory index for context injection const memoryIndex = await sessionManager.buildMemoryIndex(tenantId, userId, sessionId); - -// Get the session directory path (synchronous) -const dir = sessionManager.getSessionDir(tenantId, sessionId); ``` These are useful when building session management UIs or context provider implementations. @@ -123,23 +120,16 @@ import type { Message, Usage } from "@agentrail/core"; export class DatabaseSessionStore implements AgentrailSessionStore { async getOrCreate(tenantId, userId, agentId, sessionId?) { - // Find or create a session row in your DB const id = sessionId ?? crypto.randomUUID(); await db.sessions.upsert({ id, tenantId, userId, agentId }); return { sessionId: id }; } - getSessionDir(tenantId, sessionId) { - // Return a logical path or temp dir for sandbox/attachment use - return `/tmp/sessions/${tenantId}/${sessionId}`; - } - async loadMessages(tenantId, sessionId, limit?) { return db.messages.findMany({ sessionId, limit }); } async loadMessagesWithBudget(tenantId, sessionId, tokenBudget?) { - // Load recent messages that fit within the token budget const all = await db.messages.findMany({ sessionId, orderBy: "desc" }); return trimToTokenBudget(all, tokenBudget); } @@ -157,7 +147,6 @@ export class DatabaseSessionStore implements AgentrailSessionStore { } async compactIfNeeded(tenantId, sessionId, summarizeFn, options?) { - // Load full history, check token count, call summarizeFn if needed const messages = await this.loadAllMessages(tenantId, sessionId); if (estimateTokens(messages) < (options?.triggerTokens ?? 80_000)) { return false; @@ -169,7 +158,67 @@ export class DatabaseSessionStore implements AgentrailSessionStore { } ``` -The minimum required methods are all eight listed above. The most frequently called are `loadMessagesWithBudget`, `appendMessages`, `recordTurn`, and `compactIfNeeded`. +The seven methods above are the required baseline. The most frequently called are `loadMessagesWithBudget`, `appendMessages`, `recordTurn`, and `compactIfNeeded`. + +### Adding memo document and sandbox support + +Stores that also implement the optional memo and tool-result methods unlock agent memory tools (`write_notes`, `write_todo`) and full `/workspace/memo/**` access inside sandboxes: + +```ts +// In DatabaseSessionStore (or a subclass): + +async readMemoryDocument(tenantId, ownerId, scope, name) { + return db.memoDocuments.findOne({ tenantId, ownerId, scope, name }) ?? null; +} + +async writeMemoryDocument(tenantId, ownerId, scope, name, content) { + await db.memoDocuments.upsert({ tenantId, ownerId, scope, name, content }); +} + +async appendMemoryDocument(tenantId, ownerId, scope, name, content) { + const existing = (await this.readMemoryDocument(tenantId, ownerId, scope, name)) ?? ""; + await this.writeMemoryDocument(tenantId, ownerId, scope, name, existing + content); +} + +async readToolResultArtifact(sessionRef, toolCallId) { + return db.toolResultArtifacts.findOne({ sessionRef, toolCallId }) ?? null; +} + +async writeToolResultArtifact(sessionRef, toolCallId, content) { + await db.toolResultArtifacts.upsert({ sessionRef, toolCallId, content }); +} + +async listToolResultArtifactIds(sessionRef) { + return db.toolResultArtifacts.findIds({ sessionRef }); +} +``` + +Pass the same store instance as `memoProvider` on `SandboxManagerOptions` so the `SandboxManager` can snapshot memo documents and tool-result artifacts into the container at creation time, and write agent-side edits back to the store: + +```ts +import { SandboxManager } from "@agentrail/capabilities"; + +const store = new DatabaseSessionStore(); + +const sandboxManager = new SandboxManager( + process.env.AGENTRAIL_DATA_DIR!, + { memoProvider: store }, // read + write-back of /workspace/memo/** paths +); + +const app = createAgentApp({ + sessionStore: store, + sandboxManager, + profiles: [defaultProfile], +}); +``` + +When `memoProvider` is set, the `SandboxManager`: + +1. Snapshots memo documents and all stored tool-result artifacts into a temporary host directory before container creation. +2. Bind-mounts that directory to `/workspace/memo/` in the container as **read-only** — Bash cannot bypass the structured Write/Edit tools to write memo files. +3. Propagates agent `Write` / `Edit` writes to memo paths back to the store via `writeMemoryDocument` / `writeToolResultArtifact`. + +A full PostgreSQL implementation is available out of the box via `@agentrail/storage-postgres`. See [Session Store Reference](../reference/session-store.md) for the complete interface. ## Horizontal Scaling Considerations diff --git a/docs/public/llms.txt b/docs/public/llms.txt index 73d25ad..6b7cfe9 100644 --- a/docs/public/llms.txt +++ b/docs/public/llms.txt @@ -30,9 +30,11 @@ - [Write a Plugin](https://agentrail.run/guides/write-a-plugin): Extend the hosted lifecycle with plugins - [Multi-Agent](https://agentrail.run/guides/multi-agent): Use orchestration and sub-agents - [Configure Sessions](https://agentrail.run/guides/configure-sessions): Configure session persistence and compaction +- [Build a Storage Backend](https://agentrail.run/guides/build-a-storage-backend): Implement all storage contracts for a custom backend (DB, Redis, etc.) - [Consume Stream](https://agentrail.run/guides/consume-stream): Handle SSE events from `/stream` - [Deployment](https://agentrail.run/guides/deployment): Deploy the server and docs examples - [Troubleshooting](https://agentrail.run/guides/troubleshooting): Fix common runtime and environment problems +- [Tool Permissions](https://agentrail.run/guides/tool-permissions): Configure allow/deny/ask policies for tool execution - [Use Capability Packages](https://agentrail.run/guides/use-capability-packages): Compose capabilities into hosted profiles - [Use OpenAI-Compatible Providers](https://agentrail.run/guides/use-openai-compatible-providers): Point Agentrail at compatible LLM APIs @@ -49,6 +51,30 @@ - [Telemetry Sink](https://agentrail.run/reference/telemetry-sink): Telemetry sink contract and default sinks - [Inspector Route](https://agentrail.run/reference/inspector-route): Read-only inspection API +## Built-in tools + +- [Tools Index](https://agentrail.run/tools/): Overview of built-in hosted and orchestration tools +- [Ask User Question](https://agentrail.run/tools/ask-user-question): Suspend execution and wait for a user reply +- [Bash](https://agentrail.run/tools/bash): Run shell commands in the host or sandbox +- [Browser Action](https://agentrail.run/tools/browser-action): Click, type, and interact with web pages +- [Browser Content](https://agentrail.run/tools/browser-content): Read the current page content from the browser +- [Browser Navigate](https://agentrail.run/tools/browser-navigate): Open URLs and move the browser to a new page +- [Browser Scroll](https://agentrail.run/tools/browser-scroll): Scroll the active browser page +- [Close Agent](https://agentrail.run/tools/close-agent): Close a managed sub-agent +- [Edit](https://agentrail.run/tools/edit): In-place string replacement for files +- [Glob](https://agentrail.run/tools/glob): Match files by glob pattern +- [Grep](https://agentrail.run/tools/grep): Search file contents with ripgrep-style matching +- [Python](https://agentrail.run/tools/python): Run Python code in the sandbox and capture outputs +- [Read](https://agentrail.run/tools/read): Read file contents from the host or sandbox +- [Send Input](https://agentrail.run/tools/send-input): Send follow-up input to an existing managed agent +- [Sleep](https://agentrail.run/tools/sleep): Pause execution for a bounded duration +- [Spawn Agent](https://agentrail.run/tools/spawn-agent): Create a managed sub-agent +- [TodoWrite](https://agentrail.run/tools/todo-write): Maintain the session TODO list +- [Wait Agent](https://agentrail.run/tools/wait-agent): Wait for a managed agent to finish +- [Web Fetch](https://agentrail.run/tools/web-fetch): Fetch and summarize a web page +- [Web Search](https://agentrail.run/tools/web-search): Search the web and return concise results +- [Write](https://agentrail.run/tools/write): Write a file in the host or sandbox + ## Examples - [Playground Server](https://agentrail.run/examples/playground-server): Full hosted app example diff --git a/docs/reference/create-agent-app.md b/docs/reference/create-agent-app.md index 5672844..d158d65 100644 --- a/docs/reference/create-agent-app.md +++ b/docs/reference/create-agent-app.md @@ -188,27 +188,95 @@ Sandbox manager for upload handling and workspace snapshots in the `/stream` rou --- +### `traceStoreFactory` + +```ts +traceStoreFactory?: (sessionRef: SessionRef) => SessionTraceStore +``` + +Factory that creates a session-scoped trace store for each request. Trace envelopes (agent loop events, tool calls, compaction events) are written via `store.appendEnvelope(envelope)`. + +When omitted and `dataDir` is set, `createAgentApp` falls back to the default filesystem-backed `createFileSystemSessionTraceStore`. When neither is provided, traces are not persisted (but still forwarded to `telemetrySink` if one is configured). + +```ts +import { createAgentApp, createFileSystemSessionTraceStore } from "@agentrail/app"; +import { PostgresSessionTraceStore } from "@agentrail/storage-postgres"; + +// PostgreSQL example +const app = createAgentApp({ + sessionStore: new PostgresSessionStore(sql), + profiles: [myProfile], + traceStoreFactory: (sessionRef) => new PostgresSessionTraceStore(sql, sessionRef), +}); +``` + +--- + +### `createOrchestrationPersistence` + +```ts +createOrchestrationPersistence?: (sessionRef: SessionRef) => OrchestrationPersistence +``` + +Factory that creates an `OrchestrationPersistence` for each session. When provided (and `orchestrationRegistry` is not), `createAgentApp` automatically builds an `AgentrailOrchestrationRegistry` backed by this factory. This is the recommended shorthand for hosts that only need to swap the storage backend. + +```ts +import { createAgentApp } from "@agentrail/app"; +import { PostgresOrchestrationPersistence, createSqlClient } from "@agentrail/storage-postgres"; + +const sql = createSqlClient({ connectionString: process.env.DATABASE_URL! }); + +const app = createAgentApp({ + sessionStore: new PostgresSessionStore(sql), + profiles: [myProfile], + createOrchestrationPersistence: (sessionRef) => + new PostgresOrchestrationPersistence(sql, sessionRef), +}); +``` + +When omitted, orchestration SSE events are not forwarded unless an explicit `orchestrationRegistry` is passed. + +--- + ### `orchestrationRegistry` ```ts orchestrationRegistry?: AgentrailOrchestrationRegistry ``` -Registry instance used by the `/stream` route to subscribe to sub-agent events for real-time SSE forwarding. Pass the same registry instance you used when wiring up `orchestration(registry, factory)` inside your profile. +Explicit registry instance used by the `/stream` route to subscribe to sub-agent events for real-time SSE forwarding. Pass the same registry instance you used when wiring up `orchestration(registry, factory)` inside your profile. -When omitted, orchestration SSE events are not forwarded to the client. +Use `orchestrationRegistry` when you need direct control over the registry (e.g. to call `invalidate()` or share it with other parts of the server). For simpler setups, use `createOrchestrationPersistence` instead and let `createAgentApp` build the registry. + +When neither is provided, orchestration SSE events are not forwarded to the client. --- ### `inspector` ```ts -inspector?: true +inspector?: true | InspectorDataSource ``` -Set to `true` to mount the read-only Inspector API at `/__inspector`. This API is consumed by the [Agentrail Inspector](https://github.com/yai-dev/agentrail-inspector) Docker image. +Mounts the read-only Inspector API at `/__inspector`, consumed by the [Agentrail Inspector](https://github.com/yai-dev/agentrail-inspector) Docker image. + +Three variants: -**Requires `dataDir`** — incompatible with a custom `sessionStore`. An error is thrown at startup when this invariant is violated. +- **`true`** — automatically creates a filesystem data source from `dataDir`. **Requires `dataDir`** and is incompatible with a custom `sessionStore`. An error is thrown at startup if either constraint is violated. +- **`InspectorDataSource`** — use a custom data source (e.g. a PostgreSQL backend). Works with any `sessionStore` configuration. + +```ts +// Filesystem backend (default) +createAgentApp({ dataDir: "./data", profiles: [...], inspector: true }); + +// Custom backend +import { PostgresInspectorDataSource } from "@agentrail/storage-postgres"; +createAgentApp({ + sessionStore: new PostgresSessionStore(sql), + profiles: [...], + inspector: new PostgresInspectorDataSource(sql), +}); +``` See [Inspector Route Reference](/reference/inspector-route). diff --git a/docs/reference/inspector-route.md b/docs/reference/inspector-route.md index f151515..58f1058 100644 --- a/docs/reference/inspector-route.md +++ b/docs/reference/inspector-route.md @@ -4,13 +4,15 @@ The Inspector route exposes a read-only HTTP API consumed by the [Agentrail Insp ## Enabling the Inspector -Pass `inspector: true` to `createAgentApp`: +### Filesystem backend (default) + +Pass `inspector: true` to `createAgentApp`. Requires `dataDir` to be set: ```ts import { createAgentApp } from "@agentrail/app"; const app = createAgentApp({ - dataDir: "./data", // required when inspector is enabled + dataDir: "./data", profiles: [myProfile], inspector: true, }); @@ -18,10 +20,29 @@ const app = createAgentApp({ The route is mounted at the fixed path `/__inspector`. The Agentrail Inspector Docker image's nginx proxy hardcodes this prefix, so the mount path is not configurable in v1. -**Requirements:** +### Custom data source (database backend) + +Pass an `InspectorDataSource` object as the `inspector` option: + +```ts +import { createAgentApp } from "@agentrail/app"; +import { PostgresInspectorDataSource, createSqlClient } from "@agentrail/storage-postgres"; + +const sql = createSqlClient({ connectionString: process.env.DATABASE_URL! }); + +const app = createAgentApp({ + sessionStore: new PostgresSessionStore(sql), + profiles: [myProfile], + inspector: new PostgresInspectorDataSource(sql), +}); +``` + +Any object implementing the `InspectorDataSource` interface is accepted — you can write a custom adapter for any storage backend. + +**Constraints:** -- `dataDir` must be set. A custom `sessionStore` is not supported. -- An error is thrown at startup if `inspector: true` is set without `dataDir`. +- `inspector: true` requires `dataDir` and is incompatible with a custom `sessionStore`. An error is thrown at startup if either invariant is violated. +- Passing an explicit `InspectorDataSource` works with any `sessionStore` configuration, including custom stores. ## Running the Inspector UI @@ -65,8 +86,6 @@ Returns a list of all sessions across all tenants with enriched metadata. } ``` -Metadata is derived by reading `session.jsonl` (for userId, turns, and token counts) and the trace `events.jsonl` (for `lastActive` timestamp and error detection). - --- ### `GET /sessions/:sessionId/trace` @@ -92,7 +111,7 @@ Returns the merged trace for a session: runtime events (agent loop, tools, compa "sessionId": "...", "tenantId": "default", "traceId": "...", - "event": { "type": "session.start", ... } + "event": { "type": "session.start" } } ] } @@ -173,15 +192,50 @@ The Inspector API has no built-in authentication. It exposes **all session data* ## Advanced: mounting the Inspector manually -If you are not using `createAgentApp`, you can mount the route directly using the advanced API: +If you are not using `createAgentApp`, mount the route directly via the advanced API: ```ts -import { createInspectorRoute } from "@agentrail/app/advanced"; +import { createInspectorRoute, createFilesystemInspectorDataSource } from "@agentrail/app/advanced"; import { Hono } from "hono"; const app = new Hono(); -app.route("/__inspector", createInspectorRoute("./data")); +// Filesystem backend +app.route("/__inspector", createInspectorRoute(createFilesystemInspectorDataSource("./data"))); + +// Or a custom backend +import { PostgresInspectorDataSource } from "@agentrail/storage-postgres"; +app.route("/__inspector", createInspectorRoute(new PostgresInspectorDataSource(sql))); +``` + +## Implementing a custom `InspectorDataSource` + +```ts +import type { InspectorDataSource } from "@agentrail/app/advanced"; + +export class MyInspectorDataSource implements InspectorDataSource { + async listSessions(): Promise { + /* ... */ + } + async loadTraceEnvelopes( + tenantId: string, + sessionId: string, + ): Promise { + /* ... */ + } + async loadOrchestrationEvents(tenantId: string, sessionId: string): Promise { + /* ... */ + } + async loadOrchestrationState( + tenantId: string, + sessionId: string, + ): Promise<{ snapshot: OrchestrationSnapshot } | null> { + /* ... */ + } + async loadMessages(tenantId: string, sessionId: string): Promise { + /* ... */ + } +} ``` -This is equivalent to what `createAgentApp({ inspector: true })` does internally. +Import the full interface type from `@agentrail/app/advanced`. diff --git a/docs/reference/session-store.md b/docs/reference/session-store.md index 569d462..5c20bf1 100644 --- a/docs/reference/session-store.md +++ b/docs/reference/session-store.md @@ -13,19 +13,20 @@ Read this page when: ## Responsibilities - create or resume sessions -- resolve session directories - load message history - append messages - record usage - compact history when needed +- _(optional)_ read/write memo documents (`NOTES.md`, `TODO.md`, `USER.md`) +- _(optional)_ read/write tool result artifacts -The default examples use the filesystem-backed `SessionManager` from `@agentrail/app`. +The default implementation is the filesystem-backed `SessionManager` from `@agentrail/app`. A PostgreSQL implementation is available in `@agentrail/storage-postgres`. ## Interface -The full `AgentrailSessionStore` contract defined in `packages/app/src/host/types.ts`: +The full `AgentrailSessionStore` contract defined in `@agentrail/core`: -### `getOrCreate` +### `getOrCreate` _(required)_ ```ts getOrCreate( @@ -38,15 +39,7 @@ getOrCreate( Returns an existing session or creates a new one. If `sessionId` is omitted, a new UUID is generated. -### `getSessionDir` - -```ts -getSessionDir(tenantId: string, sessionId: string): string -``` - -Returns the filesystem path (or equivalent logical path) for the session's data directory. Called synchronously by the host before agent construction. - -### `loadMessages` +### `loadMessages` _(required)_ ```ts loadMessages(tenantId: string, sessionId: string, limit?: number): Promise @@ -54,7 +47,7 @@ loadMessages(tenantId: string, sessionId: string, limit?: number): Promise @@ -74,7 +67,7 @@ loadAllMessages(tenantId: string, sessionId: string): Promise Returns the complete, unbounded message history for a session. Used by the compaction system to decide whether to summarize old turns. -### `appendMessages` +### `appendMessages` _(required)_ ```ts appendMessages(tenantId: string, sessionId: string, messages: Message[]): Promise @@ -82,7 +75,7 @@ appendMessages(tenantId: string, sessionId: string, messages: Message[]): Promis Persists new messages to the session store after a turn completes. -### `recordTurn` +### `recordTurn` _(required)_ ```ts recordTurn(tenantId: string, sessionId: string, usage: Usage): Promise @@ -90,7 +83,7 @@ recordTurn(tenantId: string, sessionId: string, usage: Usage): Promise Records token usage for the turn. Used for billing or observability. -### `compactIfNeeded` +### `compactIfNeeded` _(required)_ ```ts compactIfNeeded( @@ -108,15 +101,93 @@ compactIfNeeded( Runs compaction if the accumulated history exceeds `triggerTokens`. Calls `summarizeFn` to collapse old messages into a summary message and persists the compacted history. Returns `true` if compaction ran. +### `readMemoryDocument` _(optional)_ + +```ts +readMemoryDocument?( + tenantId: string, + ownerId: string, + scope: "session" | "user", + name: "NOTES.md" | "TODO.md" | "USER.md", +): Promise +``` + +Reads a named memo document for a session or user. Called by the user-memory consolidation service and by context providers that inject memory into the agent's context. + +**Required** when using `UserMemoryConsolidationService` with a non-filesystem store. `SessionManager` implements this automatically. If your store omits this method and `UserMemoryConsolidationService` is configured, the service will throw at runtime. + +### `writeMemoryDocument` _(optional)_ + +```ts +writeMemoryDocument?( + tenantId: string, + ownerId: string, + scope: "session" | "user", + name: "NOTES.md" | "TODO.md" | "USER.md", + content: string, +): Promise +``` + +Writes a memo document. Counterpart to `readMemoryDocument`. Same requirements apply. + +### `appendMemoryDocument` _(optional)_ + +```ts +appendMemoryDocument?( + tenantId: string, + ownerId: string, + scope: "session" | "user", + name: "NOTES.md" | "TODO.md" | "USER.md", + content: string, +): Promise +``` + +Appends to an existing memo document. Used by in-context memo tools (`write_notes`, `write_todo`). + +### `readToolResultArtifact` _(optional)_ + +```ts +readToolResultArtifact?( + sessionRef: SessionRef, + toolCallId: string, +): Promise +``` + +Reads a compacted tool result artifact. Called during history reconstruction when a tool result was stored separately to keep the message history compact. + +### `writeToolResultArtifact` _(optional)_ + +```ts +writeToolResultArtifact?( + sessionRef: SessionRef, + toolCallId: string, + content: string, +): Promise +``` + +Writes a compacted tool result artifact. Called by `compactToolResults` when tool outputs are too large to keep inline. + +### `listToolResultArtifactIds` _(optional)_ + +```ts +listToolResultArtifactIds?(sessionRef: SessionRef): Promise +``` + +Returns all tool-call IDs whose artifacts are stored for this session. Used by `SandboxManager` at sandbox creation time to pre-populate the `/workspace/memo/session/tool-results/` directory so the agent can read compacted artifacts from inside the container. + +When not implemented, the tool-results directory starts empty inside the sandbox. Already-stored artifacts will not be accessible unless the session happens to use the filesystem backend (where they live at the expected path automatically). + +--- + ## Implementing a Custom Store -A custom store must implement all methods above. The most commonly replaced parts are `loadMessages`, `loadAllMessages`, `appendMessages`, and `compactIfNeeded` — these are the methods the host calls on every request path. +A custom store must implement all **required** methods. The optional methods unlock additional features (memo documents, tool result compaction). You can omit optional methods and add them incrementally as needed. -Below is a minimal in-memory implementation that satisfies the full interface. Use it as a starting point before wiring up a real database backend: +### Minimal in-memory example ```ts import { randomUUID } from "node:crypto"; -import type { Message, Usage } from "@agentrail/core"; +import type { Message, SessionRef, Usage } from "@agentrail/core"; import type { AgentrailSessionStore } from "@agentrail/app"; interface SessionRecord { @@ -129,9 +200,9 @@ export class InMemorySessionStore implements AgentrailSessionStore { private readonly sessions = new Map(); async getOrCreate( - tenantId: string, - userId: string, - agentId: string, + _tenantId: string, + _userId: string, + _agentId: string, sessionId?: string, ): Promise<{ sessionId: string }> { const id = sessionId ?? randomUUID(); @@ -141,29 +212,18 @@ export class InMemorySessionStore implements AgentrailSessionStore { return { sessionId: id }; } - getSessionDir(tenantId: string, sessionId: string): string { - // Return a logical path — the in-memory store doesn't use the filesystem, - // but the host calls this synchronously before agent construction. - return `/tmp/sessions/${tenantId}/${sessionId}`; - } - - async loadMessages(tenantId: string, sessionId: string, limit?: number): Promise { - const session = this.sessions.get(sessionId); - if (!session) return []; - const msgs = session.messages; + async loadMessages(_tenantId: string, sessionId: string, limit?: number): Promise { + const msgs = this.sessions.get(sessionId)?.messages ?? []; return limit ? msgs.slice(-limit) : msgs.slice(-50); } async loadMessagesWithBudget( - tenantId: string, + _tenantId: string, sessionId: string, tokenBudget?: number, ): Promise { - const session = this.sessions.get(sessionId); - if (!session) return []; - // Simplified: estimate ~4 chars per token; trim from the front + const messages = this.sessions.get(sessionId)?.messages ?? []; const budget = tokenBudget ?? 100_000; - const messages = [...session.messages]; let totalChars = 0; const result: Message[] = []; for (let i = messages.length - 1; i >= 0; i--) { @@ -175,33 +235,25 @@ export class InMemorySessionStore implements AgentrailSessionStore { return result; } - async loadAllMessages(tenantId: string, sessionId: string): Promise { + async loadAllMessages(_tenantId: string, sessionId: string): Promise { return this.sessions.get(sessionId)?.messages ?? []; } - async appendMessages(tenantId: string, sessionId: string, messages: Message[]): Promise { + async appendMessages(_tenantId: string, sessionId: string, messages: Message[]): Promise { const session = this.sessions.get(sessionId); - if (session) { - session.messages.push(...messages); - } + if (session) session.messages.push(...messages); } - async recordTurn(tenantId: string, sessionId: string, usage: Usage): Promise { + async recordTurn(_tenantId: string, sessionId: string, usage: Usage): Promise { const session = this.sessions.get(sessionId); - if (session) { - session.usageHistory.push(usage); - } + if (session) session.usageHistory.push(usage); } async compactIfNeeded( - tenantId: string, + _tenantId: string, sessionId: string, summarizeFn: (messages: Message[]) => Promise, - options?: { - triggerTokens?: number; - compactFraction?: number; - preloadedMessages?: Message[]; - }, + options?: { triggerTokens?: number; compactFraction?: number; preloadedMessages?: Message[] }, ): Promise { const session = this.sessions.get(sessionId); if (!session) return false; @@ -210,22 +262,16 @@ export class InMemorySessionStore implements AgentrailSessionStore { const compactFraction = options?.compactFraction ?? 0.5; const messages = options?.preloadedMessages ?? session.messages; - // Estimate token count (rough: 4 chars ≈ 1 token) const estimatedTokens = JSON.stringify(messages).length / 4; if (estimatedTokens < triggerTokens) return false; - // Summarize the oldest fraction of messages const cutoff = Math.floor(messages.length * compactFraction); - const oldMessages = messages.slice(0, cutoff); - const recentMessages = messages.slice(cutoff); - - const summary = await summarizeFn(oldMessages); + const summary = await summarizeFn(messages.slice(0, cutoff)); session.messages = [ { role: "user", content: `[Conversation summary]: ${summary}` }, - ...recentMessages, + ...messages.slice(cutoff), ]; - return true; } } @@ -243,19 +289,23 @@ const app = createAgentApp({ }); ``` -Or with route primitives directly: +### Production database backend + +For a production database-backed implementation, replace the `Map` with queries to your database in `loadMessages`, `appendMessages`, and `compactIfNeeded`. The interface is intentionally small so each method maps cleanly to one or two queries. + +A full PostgreSQL implementation is available out of the box: ```ts -import { createStreamRoute } from "@agentrail/app/advanced"; -import { InMemorySessionStore } from "./in-memory-session-store.js"; +import { PostgresSessionStore, createSqlClient } from "@agentrail/storage-postgres"; -app.route( - "/api/stream", - createStreamRoute({ - sessionStore: new InMemorySessionStore(), - // ... - }), -); +const sql = createSqlClient({ connectionString: process.env.DATABASE_URL! }); + +const app = createAgentApp({ + sessionStore: new PostgresSessionStore(sql), + traceStoreFactory: (sessionRef) => new PostgresSessionTraceStore(sql, sessionRef), + inspector: new PostgresInspectorDataSource(sql), + profiles: [defaultProfile], +}); ``` -For a production database-backed implementation, replace the `Map` with queries to your database in `loadMessages`, `appendMessages`, and `compactIfNeeded`. The interface is intentionally small so each method maps cleanly to one or two queries. +`PostgresSessionStore` implements all required methods plus the optional memo document and tool result artifact methods, so all host features work without additional configuration. diff --git a/docs/tools/bash.md b/docs/tools/bash.md index c451d66..d71cf1f 100644 --- a/docs/tools/bash.md +++ b/docs/tools/bash.md @@ -78,6 +78,7 @@ See [Tool Permissions](../guides/tool-permissions.md) for the full DSL reference - Do not use `Bash` for file search — use `Grep` instead. - Set `timeout: 0` for long-running processes (dev servers, watchers) that should run in the background. - Output is truncated at 1 MB. +- **`/workspace/memo/**`is read-only for Bash.** Shell writes to memo paths (e.g.`echo ... > /workspace/memo/session/NOTES.md`) will fail with a permission error. Use the `Write`or`Edit` tools to persist memo changes — they write back to the backing store regardless of which storage backend is configured. ## Example @@ -95,7 +96,7 @@ When the sandbox capability is active, a sandboxed `Bash` tool replaces the host - Commands run inside an isolated Docker container. - The working directory must be inside `/workspace` (defaults to `/workspace`). -- Session memo files are at `/workspace/memo/session/`; user profile at `/workspace/memo/user/USER.md`. +- Session memo files are at `/workspace/memo/session/`; user profile at `/workspace/memo/user/USER.md`. These paths are **read-only** for Bash — writes must go through the `Write` or `Edit` tools. - No `isDangerousCommand` check — the sandbox provides isolation instead. - Background mode does not return a `pid` (the process runs inside the container). diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..8effcb1 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,16 @@ +# Agentrail Examples + +This directory contains example applications that demonstrate how to use Agentrail. + +## Provided Examples + +- [`playground-server`](./playground-server/README.md): full hosted reference server +- [`playground-ui`](./playground-ui/README.md): companion UI for the playground server +- [`deep-research`](./deep-research/README.md): deep research workflow example +- [`minimal-agent-server`](./minimal-agent-server/README.md): minimal example of an Agentrail server +- [`multiple-profiles`](./multiple-profiles/README.md): example of using multiple profiles in an Agentrail server +- [`pg-backend-agent-server`](./pg-backend-agent-server/README.md): example of using an Agentrail server with a PostgreSQL backend +- [`with-custom-hooks`](./with-custom-hooks/README.md): example of using custom hooks in an Agentrail server +- [`with-custom-tools`](./with-custom-tools/README.md): example of using custom tools in an Agentrail server +- [`with-custom-skills`](./with-custom-skills/README.md): example of using custom skills in an Agentrail server +- [`with-custom-plugins`](./with-custom-plugins/README.md): example of using custom plugins in an Agentrail server diff --git a/examples/playground-server/src/context/index.ts b/examples/playground-server/src/context/index.ts index 5bb99b1..9f3ed0c 100644 --- a/examples/playground-server/src/context/index.ts +++ b/examples/playground-server/src/context/index.ts @@ -15,6 +15,7 @@ export const knowledgeManager = new KnowledgeManager(config.dataDir); export const skillManager = new SkillManager(config.dataDir); export const sandboxManager = new SandboxManager(config.dataDir, config.sandbox); export const userMemoryConsolidationService = new UserMemoryConsolidationService( + sessionManager, sessionManager, config.dataDir, { @@ -61,5 +62,12 @@ export const listWorkspaceSnapshot = (sessionId: string) => () => sandboxManager.listWorkspace(sessionId); export const compactMessages = ( messages: Parameters[0], - ctx?: { sessionDir?: string }, -) => compactToolResults(messages, { sessionDir: ctx?.sessionDir }); + ctx?: { + writeToolResultArtifact?: (toolCallId: string, content: string) => Promise; + sessionDir?: string; + }, +) => + compactToolResults(messages, { + writeToolResultArtifact: ctx?.writeToolResultArtifact, + sessionDir: ctx?.sessionDir, + }); diff --git a/examples/playground-server/src/main.ts b/examples/playground-server/src/main.ts index 1981096..31f3d76 100644 --- a/examples/playground-server/src/main.ts +++ b/examples/playground-server/src/main.ts @@ -19,7 +19,7 @@ import { sessions } from "@/routes/sessions.js"; import { stream } from "@/routes/stream.js"; import { trace } from "@/routes/trace.js"; import { runPluginLifecycle, type PluginErrorHandler } from "@agentrail/app"; -import { createInspectorRoute } from "@agentrail/app/advanced"; +import { createFilesystemInspectorDataSource, createInspectorRoute } from "@agentrail/app/advanced"; import { serve } from "@hono/node-server"; import { Hono } from "hono"; import { logger } from "hono/logger"; @@ -49,7 +49,10 @@ app.route("/api/sessions", deepResearch); app.route("/api/sessions", trace); app.route("/api/knowledge", knowledge); -app.route("/__inspector", createInspectorRoute(config.dataDir)); +app.route( + "/__inspector", + createInspectorRoute(createFilesystemInspectorDataSource(config.dataDir)), +); const onPluginError: PluginErrorHandler = ({ plugin, hook, error }) => { console.warn(`[plugin] "${plugin}" threw in ${hook}:`, error); diff --git a/examples/playground-server/src/profiles/default-profile.ts b/examples/playground-server/src/profiles/default-profile.ts index 690d0c8..11b1abd 100644 --- a/examples/playground-server/src/profiles/default-profile.ts +++ b/examples/playground-server/src/profiles/default-profile.ts @@ -67,7 +67,18 @@ export const defaultProfile = defineProfile({ }, listSkills: () => skillManager.listSkills(), listWorkspaceSnapshot: (ctx) => sandboxManager.listWorkspace(ctx.sessionId), - compactMessages: (msgs, ctx) => compactToolResults(msgs, { sessionDir: ctx?.sessionDir }), + writeToolResultArtifact: async (ctx, toolCallId, content) => { + await Promise.all([ + sessionManager.writeToolResultArtifact?.(ctx.sessionRef, toolCallId, content), + sandboxManager.refreshMemoMirror( + ctx.sessionId, + `/workspace/memo/session/tool-results/${toolCallId}.txt`, + content, + ), + ]); + }, + compactMessages: (msgs, ctx) => + compactToolResults(msgs, { writeToolResultArtifact: ctx?.writeToolResultArtifact }), delegateSkillsToSubAgent: config.skillDelegateToSubAgent, }, { cacheTtlMs: 5_000 }, diff --git a/examples/playground-server/src/routes/stream.ts b/examples/playground-server/src/routes/stream.ts index aa1da82..fa7fdd2 100644 --- a/examples/playground-server/src/routes/stream.ts +++ b/examples/playground-server/src/routes/stream.ts @@ -14,9 +14,28 @@ import { waitHandleRegistry } from "@/wait-handle-registry.js"; import type { WorkflowTraceEventEnvelope } from "@agentrail/app"; import { createFileSystemSessionTraceStore } from "@agentrail/app"; import { createStreamRoute } from "@agentrail/app/advanced"; +import type { SessionRef } from "@agentrail/core"; const summarize = buildSummarizeFn(); +// Cache trace stores by sessionRef so we open the trace file once per session +// rather than on every envelope. +const traceStoreCache = new Map< + SessionRef, + ReturnType> +>(); +function getTraceStore(sessionRef: SessionRef) { + let store = traceStoreCache.get(sessionRef); + if (!store) { + store = createFileSystemSessionTraceStore( + config.dataDir, + sessionRef, + ); + traceStoreCache.set(sessionRef, store); + } + return store; +} + const stream = createStreamRoute({ dataDir: config.dataDir, defaultAgentId: DEFAULT_AGENT_ID, @@ -43,11 +62,7 @@ const stream = createStreamRoute({ }, }), onTraceEvent: (ctx, envelope) => { - const traceStore = createFileSystemSessionTraceStore( - config.dataDir, - ctx.sessionRef, - ); - void traceStore.appendEnvelope(envelope); + void getTraceStore(ctx.sessionRef).appendEnvelope(envelope); }, }); diff --git a/packages/app/src/advanced.ts b/packages/app/src/advanced.ts index 3e69c9b..5dc5c08 100644 --- a/packages/app/src/advanced.ts +++ b/packages/app/src/advanced.ts @@ -38,4 +38,6 @@ export { runPluginLifecycle } from "@/host/plugins.js"; /** @deprecated Use `createStaticProfileResolver` from `@agentrail/app` instead. */ export { createProfileResolver } from "@/host/profile-registry.js"; +export { createFilesystemInspectorDataSource } from "@/inspector/data-source.js"; +export type { InspectorDataSource } from "@/inspector/data-source.js"; export { createInspectorRoute } from "@/inspector/index.js"; diff --git a/packages/app/src/app/create-agent-app.ts b/packages/app/src/app/create-agent-app.ts index 054c435..db5dda9 100644 --- a/packages/app/src/app/create-agent-app.ts +++ b/packages/app/src/app/create-agent-app.ts @@ -7,16 +7,24 @@ import { runCapabilityCompatibilityChecks } from "@/compat/capability-check.js"; import type { WorkflowTraceEventEnvelope } from "@/events/index.js"; import type { ReadinessCheck } from "@/health/index.js"; import { createHealthRoute } from "@/health/index.js"; -import type { AgentrailOrchestrationRegistry } from "@/host/orchestration-registry.js"; +import { + createOrchestrationRegistry, + type AgentrailOrchestrationRegistry, +} from "@/host/orchestration-registry.js"; import type { ProfileResolver } from "@/host/profile-registry.js"; import { createStaticProfileResolver } from "@/host/profile-registry.js"; import type { ReactiveCompactionConfig, SummarizeMessagesFn } from "@/host/reactive-compaction.js"; import type { AgentrailPlugin, ContextProvider, PluginErrorHandler } from "@/host/types.js"; +import { + createFilesystemInspectorDataSource, + type InspectorDataSource, +} from "@/inspector/data-source.js"; import { createInspectorRoute } from "@/inspector/index.js"; import type { ProfileDefinition } from "@/profile/define-profile.js"; import { createChatRoute } from "@/routes/chat-route.js"; import { createStreamRoute } from "@/routes/stream-route.js"; import { SessionManager } from "@/session/session-manager.js"; +import { createFileSystemSessionTraceStore } from "@/session/trace-store.js"; import type { TelemetrySink } from "@/telemetry/sink.js"; import type { SandboxManager, ToolPermissionPolicy } from "@agentrail/capabilities"; import type { AgentrailSessionStore, Message, SessionRef } from "@agentrail/core"; @@ -95,12 +103,49 @@ export interface CreateAgentAppOptions { * Static context providers prepended to every request. */ contextProviders?: ContextProvider[]; + /** + * Factory that creates a session-scoped trace store for each request. + * + * When omitted and `dataDir` is provided, `createAgentApp` falls back to + * the default filesystem-backed `createFileSystemSessionTraceStore`. + * Provide this option to redirect workflow traces to a custom backend + * (e.g. a database or a remote tracing service). + * + * @example + * ```ts + * import { createFileSystemSessionTraceStore } from "@agentrail/app"; + * createAgentApp({ + * traceStoreFactory: (sessionRef) => + * createFileSystemSessionTraceStore(dataDir, sessionRef), + * }); + * ``` + */ + traceStoreFactory?: ( + sessionRef: import("@agentrail/core").SessionRef, + ) => import("@/session/trace-store.js").SessionTraceStore< + import("@/events/index.js").WorkflowTraceEventEnvelope + >; /** * Sandbox manager for upload handling and workspace snapshots in the stream route. * When omitted the `/stream` endpoint is still available but file-upload and * workspace-snapshot features are disabled. */ sandboxManager?: SandboxManager; + /** + * Factory that creates an `OrchestrationPersistence` for each session. + * + * When provided alongside `orchestrationRegistry`, the registry is expected to + * use the same factory — this option is a convenience shortcut for callers + * building a registry via `createOrchestrationRegistry({ createPersistence })`. + * + * When omitted and `dataDir` is provided, the default filesystem persistence + * is used. + * + * @see {@link https://agentrail.run/reference/host-primitives} + */ + createOrchestrationPersistence?: ( + sessionRef: import("@agentrail/core").SessionRef, + ) => import("@agentrail/capabilities").OrchestrationPersistence; /** * Orchestration registry used by the stream route to subscribe to sub-agent * events for real-time SSE forwarding. @@ -112,26 +157,33 @@ export interface CreateAgentAppOptions { */ orchestrationRegistry?: AgentrailOrchestrationRegistry; /** - * Set to `true` to mount the Inspector API at `/__inspector`. - * - * When enabled, `createAgentApp` exposes a read-only HTTP API that the - * Agentrail Inspector Docker image consumes to display sessions, traces, and - * orchestration data. + * Enable the Inspector API at `/__inspector`. * - * **Requires** `dataDir` — a custom `sessionStore` is not supported. An error - * is thrown at startup when this invariant is violated. + * Three variants: + * - `true` — automatically creates a filesystem data source from `dataDir`. + * Requires `dataDir` to be set; throws at startup if `dataDir` is absent. + * - `InspectorDataSource` — use a custom data source (e.g. a PostgreSQL backend). + * - `Record` — legacy no-op alias for `true`. * - * The mount path is fixed at `/__inspector` in v1 to stay in sync with the - * Inspector Docker image, whose nginx proxy hardcodes that prefix. + * The mount path is fixed at `/__inspector` to stay in sync with the + * Agentrail Inspector Docker image, whose nginx proxy hardcodes that prefix. * * @example * ```ts + * // Filesystem backend (default) * createAgentApp({ dataDir: "./data", profiles: [...], inspector: true }); + * + * // Custom backend + * import { createFilesystemInspectorDataSource } from "@agentrail/app"; + * createAgentApp({ inspector: createFilesystemInspectorDataSource("./data"), ... }); * ``` * * @see {@link https://agentrail.run/reference/inspector-route} */ - inspector?: true | Record; + inspector?: + | true + | import("@/inspector/data-source.js").InspectorDataSource + | Record; /** * Health route configuration. * @@ -236,7 +288,6 @@ export function createAgentApp(options: CreateAgentAppOptions): Hono { plugins = [], contextProviders = [], sandboxManager, - orchestrationRegistry, onPluginError, telemetrySink, health: healthOptions, @@ -244,34 +295,6 @@ export function createAgentApp(options: CreateAgentAppOptions): Hono { permissionPolicy, } = options; - /** - * Adapts a TelemetrySink into the `onTraceEvent` callback shape expected by - * both chat-route and stream-route. Fire-and-forget; errors from the sink are - * swallowed so they never break the request pipeline. - */ - const makeSinkTraceHandler = telemetrySink - ? ( - ctx: { tenantId: string; sessionId: string; sessionRef: SessionRef }, - envelope: WorkflowTraceEventEnvelope, - ): void => { - const sinkEvent = { - // envelope.traceId is now always set by wrapTraceEvent() callers; the - // fallback to envelope.id is retained only for external envelopes - // constructed without the traceId argument. - traceId: envelope.traceId ?? envelope.id, - sessionId: ctx.sessionId, - tenantId: ctx.tenantId, - timestamp: envelope.timestamp, - sequence: envelope.sequence, - source: envelope.source as "runtime" | "orchestration" | "host", - event: envelope.event, - }; - void Promise.resolve(telemetrySink.emit(sinkEvent)).catch(() => { - // sink errors must not surface to callers - }); - } - : undefined; - // Per-request flush called at the end of every chat/stream request so that // sinks with internal write buffers (e.g. custom batch sinks) never // accumulate unbounded state between requests. @@ -294,6 +317,100 @@ export function createAgentApp(options: CreateAgentAppOptions): Hono { return new SessionManager(dataDir); })(); + // Resolve the orchestration registry: + // 1. Explicit registry passed by the host — use as-is. + // 2. createOrchestrationPersistence factory provided — auto-build a registry with it. + // 3. Neither provided — no orchestration; stream route won't forward SSE events. + // + // Intentionally NOT auto-creating a registry from dataDir alone: the registry + // will throw when stream-route calls getOrchestrationManager() without a prior + // orchestration() capability registration (no createManagedAgent factory). + // This would silently break non-orchestration streaming requests. + const orchestrationRegistry: AgentrailOrchestrationRegistry | undefined = (() => { + if (options.orchestrationRegistry) return options.orchestrationRegistry; + if (options.createOrchestrationPersistence) { + return createOrchestrationRegistry({ + createPersistence: options.createOrchestrationPersistence, + }); + } + return undefined; + })(); + + // Build a per-session trace-store handler for persisting trace envelopes. + // Precedence: explicit traceStoreFactory > dataDir filesystem fallback > none. + const traceStoreHandler: + | (( + ctx: { tenantId: string; sessionId: string; sessionRef: SessionRef }, + envelope: WorkflowTraceEventEnvelope, + ) => void) + | undefined = (() => { + const factory = options.traceStoreFactory; + if (factory) { + // Cache trace stores by sessionRef to avoid re-creating on every event. + const cache = new Map>(); + return ( + ctx: { tenantId: string; sessionId: string; sessionRef: SessionRef }, + envelope: WorkflowTraceEventEnvelope, + ) => { + let store = cache.get(ctx.sessionRef); + if (!store) { + store = factory(ctx.sessionRef); + cache.set(ctx.sessionRef, store); + } + void store.appendEnvelope(envelope).catch(() => {}); + }; + } + if (dataDir) { + return ( + _ctx: { tenantId: string; sessionId: string; sessionRef: SessionRef }, + envelope: WorkflowTraceEventEnvelope, + ) => { + // Note: createFileSystemSessionTraceStore lazily opens the file, so + // creating a new instance per call is safe and avoids reference leaks. + const store = createFileSystemSessionTraceStore( + dataDir, + _ctx.sessionRef, + ); + void store.appendEnvelope(envelope).catch(() => {}); + }; + } + return undefined; + })(); + + /** + * Adapts a TelemetrySink into the `onTraceEvent` callback shape expected by + * both chat-route and stream-route. Fire-and-forget; errors from the sink are + * swallowed so they never break the request pipeline. + * + * Also writes to the trace store (traceStoreHandler) when one is configured. + */ + const makeSinkTraceHandler = + traceStoreHandler || telemetrySink + ? ( + ctx: { tenantId: string; sessionId: string; sessionRef: SessionRef }, + envelope: WorkflowTraceEventEnvelope, + ): void => { + traceStoreHandler?.(ctx, envelope); + if (telemetrySink) { + const sinkEvent = { + // envelope.traceId is now always set by wrapTraceEvent() callers; the + // fallback to envelope.id is retained only for external envelopes + // constructed without the traceId argument. + traceId: envelope.traceId ?? envelope.id, + sessionId: ctx.sessionId, + tenantId: ctx.tenantId, + timestamp: envelope.timestamp, + sequence: envelope.sequence, + source: envelope.source as "runtime" | "orchestration" | "host", + event: envelope.event, + }; + void Promise.resolve(telemetrySink.emit(sinkEvent)).catch(() => { + // sink errors must not surface to callers + }); + } + } + : undefined; + const defaultAgentId = explicitDefaultAgentId ?? profiles[0]?.id; if (!defaultAgentId) { throw new Error("createAgentApp: `defaultAgentId` is required when `profiles` is empty."); @@ -351,19 +468,31 @@ export function createAgentApp(options: CreateAgentAppOptions): Hono { runCapabilityCompatibilityChecks(profiles, Boolean(sandboxManager)); // ── Inspector validation ─────────────────────────────────────────────────── + let resolvedInspectorDataSource: InspectorDataSource | null = null; if (inspectorOptions) { - if (!dataDir) { - throw new Error( - "createAgentApp: `inspector` requires `dataDir` to be set. " + - "Custom sessionStore implementations are not supported by the built-in Inspector API.", - ); - } - if (options.sessionStore) { - throw new Error( - "createAgentApp: `inspector` is incompatible with a custom `sessionStore`. " + - "The Inspector API reads directly from the filesystem layout produced by the default " + - "SessionManager. Remove `sessionStore` from your options, or disable `inspector`.", - ); + const isExplicitDataSource = + typeof inspectorOptions === "object" && "listSessions" in inspectorOptions; + + if (isExplicitDataSource) { + // Explicit InspectorDataSource — always allowed. + resolvedInspectorDataSource = inspectorOptions as InspectorDataSource; + } else { + // `true` or legacy `{}` → filesystem mode; requires dataDir and no custom sessionStore. + if (!dataDir) { + throw new Error( + "createAgentApp: `inspector: true` requires `dataDir` to be set. " + + "For custom storage backends, pass an `InspectorDataSource` object instead.", + ); + } + if (options.sessionStore) { + throw new Error( + "createAgentApp: `inspector: true` is incompatible with a custom `sessionStore`. " + + "The built-in Inspector reads directly from the filesystem layout produced by " + + "SessionManager. Either remove `sessionStore` from your options, or pass an " + + "explicit `InspectorDataSource` as `inspector: `.", + ); + } + resolvedInspectorDataSource = createFilesystemInspectorDataSource(dataDir); } } @@ -375,14 +504,17 @@ export function createAgentApp(options: CreateAgentAppOptions): Hono { } // ── Inspector API ────────────────────────────────────────────────────────── - if (inspectorOptions && dataDir) { - app.route("/__inspector", createInspectorRoute(dataDir)); + if (resolvedInspectorDataSource) { + app.route("/__inspector", createInspectorRoute(resolvedInspectorDataSource)); } app.route("/chat", createChatRoute(chatRouteOptions)); // Always mount /stream. sandboxManager is optional — when absent, file-upload // and workspace-snapshot features are simply not available. + // + // dataDir is forwarded only for attachment/upload support; trace store + // persistence is handled through the combined onTraceEvent handler above. const streamRouteOptions = { ...chatRouteOptions, ...(dataDir ? { dataDir } : {}), diff --git a/packages/app/src/host/defaults/capability-messages.ts b/packages/app/src/host/defaults/capability-messages.ts index 5a544fd..747f01e 100644 --- a/packages/app/src/host/defaults/capability-messages.ts +++ b/packages/app/src/host/defaults/capability-messages.ts @@ -35,21 +35,29 @@ export function makeDateContextMessage(timestamp = Date.now()): UserMessage { }; } -/** Rewrites local memory paths into sandbox-visible paths for model context. */ +/** + * @deprecated Paths in `MemoryIndex.entries` are now canonical `/workspace/memo/**` paths. + * This function is kept for backward compatibility and will be removed in a future release. + */ export function translateMemoryPaths(index: MemoryIndex): MemoryIndex { + const sessionDir = index.sessionDir; + const userDir = index.userDir; + + if (!sessionDir && !userDir) { + return index; + } + const translatePath = (filePath: string): string => { - if (filePath.startsWith(index.sessionDir)) { - return "/workspace/memo/session" + filePath.slice(index.sessionDir.length); + if (sessionDir && filePath.startsWith(sessionDir)) { + return "/workspace/memo/session" + filePath.slice(sessionDir.length); } - if (filePath.startsWith(index.userDir)) { - return "/workspace/memo/user" + filePath.slice(index.userDir.length); + if (userDir && filePath.startsWith(userDir)) { + return "/workspace/memo/user" + filePath.slice(userDir.length); } return filePath; }; return { - sessionDir: "/workspace/memo/session", - userDir: "/workspace/memo/user", entries: index.entries.map((entry: MemoryIndexEntry) => ({ ...entry, path: translatePath(entry.path), @@ -59,12 +67,7 @@ export function translateMemoryPaths(index: MemoryIndex): MemoryIndex { /** Creates a synthetic message that summarizes session and user memory files. */ export function makeMemoryIndexMessage(index: MemoryIndex, timestamp = Date.now()): UserMessage { - const lines: string[] = [ - "[Memory Index]", - `Session dir : ${index.sessionDir}`, - `User dir : ${index.userDir}`, - "", - ]; + const lines: string[] = ["[Memory Index]", "Memo root: /workspace/memo", ""]; for (const entry of index.entries) { if (!entry.exists) { diff --git a/packages/app/src/host/defaults/shared-types.ts b/packages/app/src/host/defaults/shared-types.ts index 4f89112..8c1a0fc 100644 --- a/packages/app/src/host/defaults/shared-types.ts +++ b/packages/app/src/host/defaults/shared-types.ts @@ -108,7 +108,11 @@ export interface DefaultCapabilityContextOptions { listWorkspaceSnapshot?(): Promise; compactMessages?( messages: Message[], - ctx?: { sessionDir?: string }, + ctx?: { + writeToolResultArtifact?: (toolCallId: string, content: string) => Promise; + /** @deprecated Use `writeToolResultArtifact` instead. */ + sessionDir?: string; + }, ): Message[] | Promise; } diff --git a/packages/app/src/host/orchestration-registry.ts b/packages/app/src/host/orchestration-registry.ts index 1d4ce0b..18cb5b9 100644 --- a/packages/app/src/host/orchestration-registry.ts +++ b/packages/app/src/host/orchestration-registry.ts @@ -8,6 +8,7 @@ import { createFilesystemOrchestrationPersistence, type CreateManagedAgentInput, type ManagedAgentInstance, + type OrchestrationPersistence, type StartRunInput, } from "@agentrail/capabilities"; import type { SessionRef } from "@agentrail/core"; @@ -48,7 +49,17 @@ export interface AgentrailOrchestrationRegistry { /** Inputs required to create the default orchestration registry. */ export interface CreateOrchestrationRegistryOptions { - dataDir: string; + /** + * Directory used for filesystem-backed orchestration storage. + * Required when `createPersistence` is not provided. + */ + dataDir?: string; + /** + * Factory that creates an `OrchestrationPersistence` for a given session. + * When provided, `dataDir` is ignored for orchestration storage. + * When absent, `dataDir` must be present and a filesystem persistence is used. + */ + createPersistence?: (sessionRef: SessionRef) => OrchestrationPersistence; createStartRunInput?: ( request: Pick, ) => StartRunInput; @@ -67,7 +78,13 @@ class SessionOrchestrationRegistry implements AgentrailOrchestrationRegistry { */ private readonly runStartLocks = new Map>(); - constructor(private readonly options: CreateOrchestrationRegistryOptions) {} + constructor(private readonly options: CreateOrchestrationRegistryOptions) { + if (!options.createPersistence && !options.dataDir) { + throw new Error( + "[createOrchestrationRegistry] Either `dataDir` or `createPersistence` must be provided.", + ); + } + } async getManager(request: AgentrailOrchestrationRegistryRequest): Promise { const key = this.getKey(request.tenantId, request.sessionId); @@ -120,11 +137,11 @@ class SessionOrchestrationRegistry implements AgentrailOrchestrationRegistry { request: Pick, ): Promise { try { + const persistence = this.options.createPersistence + ? this.options.createPersistence(request.sessionRef) + : createFilesystemOrchestrationPersistence(this.options.dataDir!, request.sessionRef); return await OrchestrationManager.create({ - persistence: createFilesystemOrchestrationPersistence( - this.options.dataDir, - request.sessionRef, - ), + persistence, runtime: { createAgent: async (input) => { const createManagedAgent = this.bindings.get(key); diff --git a/packages/app/src/index.ts b/packages/app/src/index.ts index ff7efe5..1572bf5 100644 --- a/packages/app/src/index.ts +++ b/packages/app/src/index.ts @@ -24,6 +24,8 @@ export type { ReactiveCompactionConfig, SummarizeMessagesFn, } from "@/host/compaction.js"; +export { createFilesystemInspectorDataSource, createInspectorRoute } from "@/inspector/index.js"; +export type { InspectorDataSource, InspectorSessionItem } from "@/inspector/index.js"; /** Build a static profile resolver backed by a fixed list of profiles. */ export { createStaticProfileResolver } from "@/host/profile-registry.js"; @@ -33,6 +35,11 @@ export type { ProfileResolver } from "@/host/profile-registry.js"; // Session management // ============================================================================ +export { + buildCompactionMessage, + buildCompactionNotesEntry, + computeCompactionSplit, +} from "@/session/compaction-logic.js"; export { compactToolResults } from "@/session/compaction.js"; export { SessionManager, @@ -40,6 +47,8 @@ export { parseCompactionMetadata, } from "@/session/session-manager.js"; export { createFileSystemSessionTraceStore } from "@/session/trace-store.js"; +export type { SessionTraceStore } from "@/session/trace-store.js"; +export type { UserSessionLister } from "@/session/user-session-lister.js"; // ============================================================================ // Host runtime types (shared across chat and stream routes) diff --git a/packages/app/src/inspector/data-source.ts b/packages/app/src/inspector/data-source.ts new file mode 100644 index 0000000..0151f91 --- /dev/null +++ b/packages/app/src/inspector/data-source.ts @@ -0,0 +1,242 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import type { WorkflowTraceEventEnvelope } from "@/events/index.js"; +import { createFileSystemSessionTraceStore } from "@/session/trace-store.js"; +import { + createFilesystemOrchestrationPersistence, + type RecoveredOrchestrationState, +} from "@agentrail/capabilities"; +import type { Message } from "@agentrail/core"; +import { createSessionRef } from "@agentrail/core"; +import { readFile, readdir } from "node:fs/promises"; +import path from "node:path"; + +// ─── Public contract ────────────────────────────────────────────────────────── + +/** Lightweight session descriptor returned by `listSessions`. */ +export interface InspectorSessionItem { + tenantId: string; + sessionId: string; + userId?: string; + lastActive?: string; + turns?: number; + tokens?: number; + status?: "idle" | "error"; +} + +/** + * Read-side abstraction consumed by `createInspectorRoute`. + * + * The default filesystem implementation (`createFilesystemInspectorDataSource`) + * reads directly from the layout produced by `SessionManager`. Custom storage + * backends should provide their own implementation of this interface and pass it + * to `createInspectorRoute` or `createAgentApp({ inspector: })`. + * + * @see {@link https://agentrail.run/reference/inspector-route} + */ +export interface InspectorDataSource { + /** List all sessions across all tenants, enriched with metadata. */ + listSessions(): Promise; + + /** Load the raw message history for a session. */ + loadMessages(tenantId: string, sessionId: string): Promise; + + /** Load merged runtime + orchestration trace envelopes for a session. */ + loadTraceEnvelopes(tenantId: string, sessionId: string): Promise; + + /** Load the current orchestration state snapshot for a session. */ + loadOrchestrationState( + tenantId: string, + sessionId: string, + ): Promise; + + /** Load the raw orchestration events for a session (for the trace timeline). */ + loadOrchestrationEvents(tenantId: string, sessionId: string): Promise; +} + +// ─── Filesystem implementation ──────────────────────────────────────────────── + +async function enrichSession( + dataDir: string, + tenantId: string, + sessionId: string, +): Promise> { + const sessionDir = path.join(dataDir, "tenants", tenantId, "sessions", sessionId); + const sessionFile = path.join(sessionDir, "session.jsonl"); + const traceFile = path.join(sessionDir, "trace", "events.jsonl"); + + let userId: string | undefined; + let turns = 0; + let tokens = 0; + + try { + const sessionRaw = await readFile(sessionFile, "utf8"); + for (const line of sessionRaw.split("\n")) { + if (!line.trim()) continue; + let record: Record; + try { + record = JSON.parse(line) as Record; + } catch { + continue; + } + const type = String(record["type"] ?? ""); + if (type === "init") { + const uid = record["userId"]; + if (typeof uid === "string" && uid) userId = uid; + else if (typeof uid === "number") userId = String(uid); + } else if (type === "turn") { + turns++; + const inp = typeof record["inputTokens"] === "number" ? record["inputTokens"] : 0; + const out = typeof record["outputTokens"] === "number" ? record["outputTokens"] : 0; + tokens += inp + out; + } + } + } catch { + // session.jsonl not yet written + } + + let lastActive: string | undefined; + let hasError = false; + let traceTurns = 0; + + try { + const traceRaw = await readFile(traceFile, "utf8"); + for (const line of traceRaw.split("\n")) { + if (!line.trim()) continue; + let envelope: Record; + try { + envelope = JSON.parse(line) as Record; + } catch { + continue; + } + const ts = envelope["timestamp"]; + if (typeof ts === "string" && (!lastActive || ts > lastActive)) { + lastActive = ts; + } + const event = (envelope["event"] ?? {}) as Record; + const type = String(event["type"] ?? ""); + if (type === "turn_end" || type === "turn.complete") traceTurns++; + if (type === "error") hasError = true; + } + } catch { + // trace not yet written + } + + const finalTurns = turns > 0 ? turns : traceTurns > 0 ? traceTurns : undefined; + + return { + userId, + lastActive, + turns: finalTurns, + tokens: tokens > 0 ? tokens : undefined, + status: hasError ? "error" : "idle", + }; +} + +/** + * Creates the default filesystem-backed `InspectorDataSource` from a `dataDir` + * that uses the layout produced by `SessionManager`. + */ +export function createFilesystemInspectorDataSource(dataDir: string): InspectorDataSource { + return { + async listSessions(): Promise { + const tenantsRoot = path.join(dataDir, "tenants"); + const items: InspectorSessionItem[] = []; + + let tenantDirs: string[]; + try { + const entries = await readdir(tenantsRoot, { withFileTypes: true }); + tenantDirs = entries.filter((e) => e.isDirectory()).map((e) => e.name); + } catch { + return []; + } + + await Promise.all( + tenantDirs.map(async (tenantId) => { + const sessionsRoot = path.join(tenantsRoot, tenantId, "sessions"); + try { + const entries = await readdir(sessionsRoot, { withFileTypes: true }); + await Promise.all( + entries + .filter((e) => e.isDirectory()) + .map(async (e) => { + const meta = await enrichSession(dataDir, tenantId, e.name); + items.push({ tenantId, sessionId: e.name, ...meta }); + }), + ); + } catch { + // no sessions for this tenant yet + } + }), + ); + + items.sort((a, b) => { + if (a.lastActive && b.lastActive) return b.lastActive.localeCompare(a.lastActive); + if (a.lastActive) return -1; + if (b.lastActive) return 1; + return a.sessionId.localeCompare(b.sessionId); + }); + + return items; + }, + + async loadMessages(tenantId: string, sessionId: string): Promise { + const file = path.join(dataDir, "tenants", tenantId, "sessions", sessionId, "messages.jsonl"); + let raw: string; + try { + raw = await readFile(file, "utf8"); + } catch { + return []; + } + return raw + .split("\n") + .filter((l) => l.trim().length > 0) + .map((l) => { + try { + return JSON.parse(l) as Message; + } catch { + return null; + } + }) + .filter((m): m is Message => m !== null); + }, + + async loadTraceEnvelopes( + tenantId: string, + sessionId: string, + ): Promise { + const sessionRef = createSessionRef(tenantId, sessionId); + const traceStore = createFileSystemSessionTraceStore( + dataDir, + sessionRef, + ); + return traceStore.loadEnvelopes(); + }, + + async loadOrchestrationState( + tenantId: string, + sessionId: string, + ): Promise { + const sessionRef = createSessionRef(tenantId, sessionId); + const persistence = createFilesystemOrchestrationPersistence(dataDir, sessionRef); + try { + return await persistence.recoverState(); + } catch { + return null; + } + }, + + async loadOrchestrationEvents(tenantId: string, sessionId: string): Promise { + const sessionRef = createSessionRef(tenantId, sessionId); + const persistence = createFilesystemOrchestrationPersistence(dataDir, sessionRef); + try { + return await persistence.loadEvents(); + } catch { + return []; + } + }, + }; +} diff --git a/packages/app/src/inspector/index.ts b/packages/app/src/inspector/index.ts index a0f2083..802fff7 100644 --- a/packages/app/src/inspector/index.ts +++ b/packages/app/src/inspector/index.ts @@ -4,190 +4,23 @@ */ import { mapOrchestrationEvent, type WorkflowTraceEventEnvelope } from "@/events/index.js"; -import { createFileSystemSessionTraceStore } from "@/session/trace-store.js"; -import { createFilesystemOrchestrationPersistence } from "@agentrail/capabilities"; -import { createSessionRef } from "@agentrail/core"; +import type { InspectorDataSource } from "@/inspector/data-source.js"; import { Hono } from "hono"; -import { readFile, readdir } from "node:fs/promises"; -import path from "node:path"; -// ─── Session listing ────────────────────────────────────────────────────────── - -interface SessionListItem { - tenantId: string; - sessionId: string; - userId?: string; - lastActive?: string; - turns?: number; - tokens?: number; - status?: "idle" | "error"; -} - -/** - * Derives session metadata by reading both session.jsonl (for userId, tokens, - * and turn count) and the trace events.jsonl (for lastActive timestamp and - * error detection). Uses session.jsonl as the authoritative source for turn - * and token counts since the trace only records agent-level events. - */ -async function enrichSession( - dataDir: string, - tenantId: string, - sessionId: string, -): Promise> { - const sessionDir = path.join(dataDir, "tenants", tenantId, "sessions", sessionId); - const sessionFile = path.join(sessionDir, "session.jsonl"); - const traceFile = path.join(sessionDir, "trace", "events.jsonl"); - - // ── Read session.jsonl for userId, tokens, and turn count ───────────────── - let userId: string | undefined; - let turns = 0; - let tokens = 0; - - try { - const sessionRaw = await readFile(sessionFile, "utf8"); - for (const line of sessionRaw.split("\n")) { - if (!line.trim()) continue; - let record: Record; - try { - record = JSON.parse(line) as Record; - } catch { - continue; - } - const type = String(record["type"] ?? ""); - if (type === "init") { - const uid = record["userId"]; - if (typeof uid === "string" && uid) userId = uid; - else if (typeof uid === "number") userId = String(uid); - } else if (type === "turn") { - turns++; - const inp = typeof record["inputTokens"] === "number" ? record["inputTokens"] : 0; - const out = typeof record["outputTokens"] === "number" ? record["outputTokens"] : 0; - tokens += inp + out; - } - } - } catch { - // session.jsonl not yet written — fall through to trace-based counting - } - - // ── Read trace events.jsonl for lastActive timestamp and errors ─────────── - let lastActive: string | undefined; - let hasError = false; - let traceTurns = 0; - - try { - const traceRaw = await readFile(traceFile, "utf8"); - for (const line of traceRaw.split("\n")) { - if (!line.trim()) continue; - let envelope: Record; - try { - envelope = JSON.parse(line) as Record; - } catch { - continue; - } - const ts = envelope["timestamp"]; - if (typeof ts === "string" && (!lastActive || ts > lastActive)) { - lastActive = ts; - } - const event = (envelope["event"] ?? {}) as Record; - const type = String(event["type"] ?? ""); - if (type === "turn_end" || type === "turn.complete") traceTurns++; - if (type === "error") hasError = true; - } - } catch { - // trace not yet written - } - - // session.jsonl turn count is authoritative; fall back to trace if missing - const finalTurns = turns > 0 ? turns : traceTurns > 0 ? traceTurns : undefined; - - return { - userId, - lastActive, - turns: finalTurns, - tokens: tokens > 0 ? tokens : undefined, - status: hasError ? "error" : "idle", - }; -} - -async function listSessions(dataDir: string): Promise { - const tenantsRoot = path.join(dataDir, "tenants"); - const items: SessionListItem[] = []; - - let tenantDirs: string[]; - try { - const entries = await readdir(tenantsRoot, { withFileTypes: true }); - tenantDirs = entries.filter((e) => e.isDirectory()).map((e) => e.name); - } catch { - return []; - } - - await Promise.all( - tenantDirs.map(async (tenantId) => { - const sessionsRoot = path.join(tenantsRoot, tenantId, "sessions"); - try { - const entries = await readdir(sessionsRoot, { withFileTypes: true }); - await Promise.all( - entries - .filter((e) => e.isDirectory()) - .map(async (e) => { - const meta = await enrichSession(dataDir, tenantId, e.name); - items.push({ tenantId, sessionId: e.name, ...meta }); - }), - ); - } catch { - // no sessions for this tenant yet - } - }), - ); - - // Sort newest-first by lastActive, then alphabetically - items.sort((a, b) => { - if (a.lastActive && b.lastActive) return b.lastActive.localeCompare(a.lastActive); - if (a.lastActive) return -1; - if (b.lastActive) return 1; - return a.sessionId.localeCompare(b.sessionId); - }); - - return items; -} - -// ─── Session messages ───────────────────────────────────────────────────────── - -async function loadSessionMessages( - dataDir: string, - tenantId: string, - sessionId: string, -): Promise { - const file = path.join(dataDir, "tenants", tenantId, "sessions", sessionId, "messages.jsonl"); - let raw: string; - try { - raw = await readFile(file, "utf8"); - } catch { - return []; - } - return raw - .split("\n") - .filter((l) => l.trim().length > 0) - .map((l) => { - try { - return JSON.parse(l) as unknown; - } catch { - return null; - } - }) - .filter(Boolean); -} +export { createFilesystemInspectorDataSource } from "@/inspector/data-source.js"; +export type { InspectorDataSource, InspectorSessionItem } from "@/inspector/data-source.js"; // ─── Route factory ──────────────────────────────────────────────────────────── /** - * Creates the Agentrail Inspector API routes. + * Creates the Agentrail Inspector API routes backed by an `InspectorDataSource`. * * The returned Hono sub-app uses **relative paths** and must be mounted by the * host at the desired prefix: * * ```ts - * app.route("/__inspector", createInspectorRoute(dataDir)); + * import { createFilesystemInspectorDataSource } from "@agentrail/app/advanced"; + * app.route("/__inspector", createInspectorRoute(createFilesystemInspectorDataSource(dataDir))); * ``` * * Exposed endpoints (all relative to the mount point): @@ -197,14 +30,11 @@ async function loadSessionMessages( * - `GET /sessions/:sessionId/messages` — raw message history (for Context Diff) * - `GET /health` — lightweight liveness check * - * **MVP limitation:** this route reads directly from the filesystem data - * layout produced by `SessionManager` and - * `createFilesystemOrchestrationPersistence`. It is not compatible with custom - * session store implementations. - * * @see {@link https://agentrail.run/reference/inspector-route} */ -export function createInspectorRoute(dataDir: string): Hono { +export function createInspectorRoute(dataSource: InspectorDataSource): Hono { + const source: InspectorDataSource = dataSource; + const route = new Hono(); // ── Liveness ────────────────────────────────────────────────────────────── @@ -213,7 +43,7 @@ export function createInspectorRoute(dataDir: string): Hono { // ── Session list (enriched) ──────────────────────────────────────────────── route.get("/sessions", async (c) => { try { - const sessions = await listSessions(dataDir); + const sessions = await source.listSessions(); return c.json({ sessions }); } catch (err) { return c.json({ error: err instanceof Error ? err.message : String(err) }, 500); @@ -224,36 +54,29 @@ export function createInspectorRoute(dataDir: string): Hono { route.get("/sessions/:sessionId/trace", async (c) => { const { sessionId } = c.req.param(); const tenantId = c.req.query("tenantId") ?? "default"; - const sessionRef = createSessionRef(tenantId, sessionId); - - const persistence = createFilesystemOrchestrationPersistence(dataDir, sessionRef); - const traceStore = createFileSystemSessionTraceStore( - dataDir, - sessionRef, - ); try { const [runtimeEnvelopes, orchestrationEvents] = await Promise.all([ - traceStore.loadEnvelopes(), - persistence.loadEvents().catch(() => []), + source.loadTraceEnvelopes(tenantId, sessionId), + source.loadOrchestrationEvents(tenantId, sessionId).catch(() => []), ]); const baseSeq = runtimeEnvelopes.length; - const orchestrationEnvelopes: WorkflowTraceEventEnvelope[] = orchestrationEvents.flatMap( - (event, i) => { - const mapped = mapOrchestrationEvent(event); - if (!mapped) return []; - return [ - { - id: `orchestration-${event.eventId}-${i}`, - timestamp: event.occurredAt, - sequence: baseSeq + i, - source: "orchestration" as const, - event: mapped as unknown as Record, - }, - ]; - }, - ); + const orchestrationEnvelopes: WorkflowTraceEventEnvelope[] = ( + orchestrationEvents as Parameters[0][] + ).flatMap((event, i) => { + const mapped = mapOrchestrationEvent(event); + if (!mapped) return []; + return [ + { + id: `orchestration-${(event as { eventId?: string }).eventId ?? i}-${i}`, + timestamp: (event as { occurredAt?: string }).occurredAt ?? new Date().toISOString(), + sequence: baseSeq + i, + source: "orchestration" as const, + event: mapped as unknown as Record, + }, + ]; + }); const merged = [...runtimeEnvelopes, ...orchestrationEnvelopes].sort((a, b) => { const tDiff = a.timestamp.localeCompare(b.timestamp); @@ -271,7 +94,7 @@ export function createInspectorRoute(dataDir: string): Hono { const { sessionId } = c.req.param(); const tenantId = c.req.query("tenantId") ?? "default"; try { - const messages = await loadSessionMessages(dataDir, tenantId, sessionId); + const messages = await source.loadMessages(tenantId, sessionId); return c.json({ messages }); } catch (err) { return c.json({ error: err instanceof Error ? err.message : String(err) }, 500); @@ -282,38 +105,37 @@ export function createInspectorRoute(dataDir: string): Hono { route.get("/sessions/:sessionId/orchestration", async (c) => { const { sessionId } = c.req.param(); const tenantId = c.req.query("tenantId") ?? "default"; - const sessionRef = createSessionRef(tenantId, sessionId); try { - const persistence = createFilesystemOrchestrationPersistence(dataDir, sessionRef); - const [{ snapshot }, events] = await Promise.all([ - persistence.recoverState(), - persistence.loadEvents(), + const [state, events] = await Promise.all([ + source.loadOrchestrationState(tenantId, sessionId), + source.loadOrchestrationEvents(tenantId, sessionId), ]); - const agents = Object.values(snapshot?.agents ?? {}); - const agentStates = await Promise.all( - agents.map(async (agent) => ({ - agent, - mailboxState: await persistence.loadMailboxState(agent.id), - })), - ); + const snapshot = state?.snapshot; + const agents = Object.values( + snapshot?.agents ?? {}, + ) as import("@agentrail/capabilities").OrchestrationAgent[]; const activeRun = snapshot - ? (Object.values(snapshot.runs).find((r) => r.status === "running") ?? - Object.values(snapshot.runs)[0]) + ? (Object.values(snapshot.runs).find( + (r) => (r as { status?: string }).status === "running", + ) ?? Object.values(snapshot.runs)[0]) : undefined; + const activeRunTyped = activeRun as + | { id: string; status: string; createdAt: string; updatedAt: string } + | undefined; return c.json({ - run: activeRun + run: activeRunTyped ? { - id: activeRun.id, - status: activeRun.status, - createdAt: activeRun.createdAt, - updatedAt: activeRun.updatedAt, + id: activeRunTyped.id, + status: activeRunTyped.status, + createdAt: activeRunTyped.createdAt, + updatedAt: activeRunTyped.updatedAt, } : null, - agents: agentStates.map(({ agent, mailboxState }) => ({ + agents: agents.map((agent) => ({ id: agent.id, displayName: agent.displayName, role: agent.role, @@ -322,10 +144,6 @@ export function createInspectorRoute(dataDir: string): Hono { updatedAt: agent.updatedAt, closedAt: agent.closedAt, lastJob: agent.lastJob, - mailbox: { - processedEventCount: mailboxState.processedEventCount, - closeRequested: mailboxState.closeRequested, - }, })), events, }); diff --git a/packages/app/src/plugins/user-memory/user-memory-consolidation-service.ts b/packages/app/src/plugins/user-memory/user-memory-consolidation-service.ts index 54b16d9..d7fe535 100644 --- a/packages/app/src/plugins/user-memory/user-memory-consolidation-service.ts +++ b/packages/app/src/plugins/user-memory/user-memory-consolidation-service.ts @@ -3,13 +3,27 @@ * Copyright (c) 2026 The Agentrail Authors */ -import { SessionManager, isCompactionMessage } from "@/session/session-manager.js"; -import type { Message } from "@agentrail/core"; +import { isCompactionMessage } from "@/session/session-manager.js"; +import type { UserSessionLister } from "@/session/user-session-lister.js"; +import type { AgentrailSessionStore, Message } from "@agentrail/core"; import { defineAgent, isRuntimeError } from "@agentrail/core"; import "@agentrail/core/providers"; import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises"; import path from "node:path"; +/** + * Minimal interface for propagating host-side user-memo updates to live + * sandbox mirrors. `SandboxManager` satisfies this interface. + */ +export interface UserMemoMirrorRefresher { + refreshUserMemoMirrorForAllSessions( + tenantId: string, + userId: string, + name: string, + content: string, + ): Promise; +} + // ============================================================================ // Constants // ============================================================================ @@ -286,9 +300,29 @@ export class UserMemoryConsolidationService { private scanning = false; constructor( - private readonly sessionManager: SessionManager, + /** + * Session store used to load session message history. + * Accepts any `AgentrailSessionStore` implementation. + */ + private readonly sessionStore: AgentrailSessionStore, + /** + * Provides user-scoped session listing for background scans. + * `SessionManager` implements this interface automatically. + */ + private readonly userSessionLister: UserSessionLister, + /** + * Host data directory used for state/summary cache files + * (`.memory-state.json`, `user-memory.json`). + */ private readonly dataDir: string, private readonly config: UserMemoryConfig, + /** + * When provided, the service propagates USER.md updates to all live + * sandbox mirrors so the agent immediately reads the updated content + * from inside any currently-running container. Pass the `SandboxManager` + * instance here when using a non-filesystem session store. + */ + private readonly mirrorRefresher?: UserMemoMirrorRefresher, ) {} // -------------------------------------------------------------------------- @@ -403,11 +437,8 @@ export class UserMemoryConsolidationService { */ private async evaluateAndQueueUser(tenantId: string, userId: string): Promise { const state = await this.readState(tenantId, userId); - const sessions = await this.sessionManager.listSessionIdsByUser( - tenantId, - userId, - MAX_SESSIONS_PER_USER, - ); + const allSessions = await this.userSessionLister.listSessionsByUser(tenantId, userId); + const sessions = allSessions.slice(0, MAX_SESSIONS_PER_USER); if (sessions.length === 0) return; const now = Date.now(); @@ -468,11 +499,8 @@ export class UserMemoryConsolidationService { forceRebuild: boolean, ): Promise { const state = await this.readState(tenantId, userId); - const sessions = await this.sessionManager.listSessionIdsByUser( - tenantId, - userId, - MAX_SESSIONS_PER_USER, - ); + const allSessions = await this.userSessionLister.listSessionsByUser(tenantId, userId); + const sessions = allSessions.slice(0, MAX_SESSIONS_PER_USER); if (sessions.length === 0) return; // Phase 1: ensure every changed session has an up-to-date cached summary. @@ -509,10 +537,7 @@ export class UserMemoryConsolidationService { continue; } - const messages = await this.sessionManager.loadFullSessionMessages( - tenantId, - session.sessionId, - ); + const messages = await this.sessionStore.loadAllMessages(tenantId, session.sessionId); // Strip compaction placeholder messages; they add noise without useful content. const usableMessages = messages.filter((m: Message) => !isCompactionMessage(m)); if (usableMessages.length < 4) continue; @@ -555,14 +580,40 @@ export class UserMemoryConsolidationService { forceRebuild: boolean, ): Promise { const profile = await this.buildUserProfile(sessionSummaries); - const userMdPath = path.join(this.sessionManager.getUserDir(tenantId, userId), "USER.md"); - const existingUserMd = await readFile(userMdPath, "utf8").catch(() => ""); + + let existingUserMd = ""; + if (this.sessionStore.readMemoryDocument) { + existingUserMd = + (await this.sessionStore.readMemoryDocument(tenantId, userId, "user", "USER.md")) ?? ""; + } else { + throw new Error( + `UserMemoryConsolidationService: the session store does not implement ` + + `readMemoryDocument. Implement this method on your store — ` + + `SessionManager implements it automatically for filesystem backends.`, + ); + } + const existingHistory = extractHistorySection(existingUserMd); const trigger = forceRebuild ? "manual" : "idle-auto"; - await atomicWrite( - userMdPath, - renderUserMd(profile, sessionSummaries, existingHistory, trigger), - ); + const newContent = renderUserMd(profile, sessionSummaries, existingHistory, trigger); + + if (this.sessionStore.writeMemoryDocument) { + await this.sessionStore.writeMemoryDocument(tenantId, userId, "user", "USER.md", newContent); + } else { + throw new Error( + `UserMemoryConsolidationService: the session store does not implement ` + + `writeMemoryDocument. Implement this method on your store — ` + + `SessionManager implements it automatically for filesystem backends.`, + ); + } + + // Refresh the live sandbox mirrors so any currently-running agent + // session immediately sees the updated USER.md inside the container. + await this.mirrorRefresher + ?.refreshUserMemoMirrorForAllSessions(tenantId, userId, "USER.md", newContent) + .catch(() => { + // Non-fatal: mirror refresh failure must not disrupt consolidation. + }); } // -------------------------------------------------------------------------- @@ -673,7 +724,7 @@ export class UserMemoryConsolidationService { // -------------------------------------------------------------------------- private getStatePath(tenantId: string, userId: string): string { - return path.join(this.sessionManager.getUserDir(tenantId, userId), STATE_FILE); + return path.join(this.dataDir, "tenants", tenantId, "users", userId, STATE_FILE); } private async readState(tenantId: string, userId: string): Promise { @@ -710,7 +761,14 @@ export class UserMemoryConsolidationService { } private getSessionSummaryPath(tenantId: string, sessionId: string): string { - return path.join(this.sessionManager.getSessionDir(tenantId, sessionId), SESSION_SUMMARY_FILE); + return path.join( + this.dataDir, + "tenants", + tenantId, + "sessions", + sessionId, + SESSION_SUMMARY_FILE, + ); } private async readSessionSummary( diff --git a/packages/app/src/session/compaction-logic.ts b/packages/app/src/session/compaction-logic.ts new file mode 100644 index 0000000..dce53da --- /dev/null +++ b/packages/app/src/session/compaction-logic.ts @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import { estimateMessageTokens } from "@/session/token-estimator.js"; +import type { Message } from "@agentrail/core"; + +/** + * Pure computation: decide which messages to compact and which to keep. + * + * Returns `null` when compaction should not run (too few messages or below the + * token threshold and `force` is false). Otherwise returns: + * - `toCompact` — the oldest slice that will be summarised + * - `toKeep` — the recent slice that stays in the visible context + * - `totalTokens` — estimated token count of the full message list + * + * No I/O is performed; all decisions are based on the provided `all` array. + */ +export function computeCompactionSplit( + all: Message[], + options: { + triggerTokens?: number; + compactFraction?: number; + force?: boolean; + } = {}, +): { toCompact: Message[]; toKeep: Message[]; totalTokens: number } | null { + const { triggerTokens = 60_000, compactFraction = 1 / 3, force = false } = options; + + if (all.length < 6) return null; + + const totalTokens = estimateMessageTokens(all); + if (!force && totalTokens <= triggerTokens) return null; + + // Advance the cut boundary past any toolResult messages to avoid orphaned + // tool results: a toolResult must always have its corresponding toolCall + // visible in the same context window. + let cutPoint = Math.max(1, Math.floor(all.length * compactFraction)); + while (cutPoint < all.length - 1 && all[cutPoint]!.role === "toolResult") { + cutPoint++; + } + // Edge case: first loop stopped at all.length-1 and it's still a toolResult + // (entire tail is toolResults). Pull back until toKeep starts on a safe boundary. + while (cutPoint > 1 && all[cutPoint]!.role === "toolResult") { + cutPoint--; + } + + return { + toCompact: all.slice(0, cutPoint), + toKeep: all.slice(cutPoint), + totalTokens, + }; +} + +/** + * Pure construction: build the synthetic compaction placeholder message. + * + * The returned message is a `user`-role message that replaces the compacted + * slice in the visible context window. No I/O is performed. + */ +export function buildCompactionMessage(params: { + summary: string; + archiveId: string; + toCompact: Message[]; + totalTokens: number; + compactFraction: number; + timestamp: string; + workspaceSnapshot?: string; +}): Message { + const { + summary, + archiveId, + toCompact, + totalTokens, + compactFraction, + timestamp, + workspaceSnapshot, + } = params; + + const lines = [ + `[Conversation compacted at ${timestamp}. Full history preserved in messages.compactions/${archiveId}.jsonl.`, + `Archive ID: ${archiveId}`, + `${toCompact.length} messages (${Math.round(totalTokens * compactFraction)} tokens estimated) were compressed.`, + ``, + `Summary of compressed conversation:`, + summary, + ]; + + if (workspaceSnapshot && workspaceSnapshot.trim().length > 0) { + lines.push(``, `Sandbox workspace at time of compaction:`, workspaceSnapshot.trim()); + } + + lines.push(`]`); + + return { + role: "user", + content: lines.join("\n"), + timestamp: Date.now(), + } as Message; +} + +/** + * Build the NOTES.md entry that records a compaction summary. + * Returned string is ready to be appended to NOTES.md (or the equivalent + * memo document) — it starts with a blank line so repeated appends are spaced. + */ +export function buildCompactionNotesEntry(params: { + summary: string; + archiveId: string; + compressedCount: number; + timestamp: string; +}): string { + const { summary, archiveId, compressedCount, timestamp } = params; + return [ + ``, + `## Compaction Summary [${timestamp}]`, + ``, + `*(${compressedCount} messages compressed, full history in \`messages.compactions/${archiveId}.jsonl\`)*`, + ``, + summary, + ``, + ].join("\n"); +} diff --git a/packages/app/src/session/compaction.ts b/packages/app/src/session/compaction.ts index 26b9365..2a9a194 100644 --- a/packages/app/src/session/compaction.ts +++ b/packages/app/src/session/compaction.ts @@ -23,7 +23,20 @@ export interface ToolResultCompactionOptions { * avoid blowing up the next model request. Default: max(maxTokensPerToolResult * 4, 6000) */ maxTokensPerRecentToolResult?: number; - /** Session directory used to persist compacted tool results for later retrieval. */ + /** + * Callback to persist a compacted tool-result artifact by tool call ID. + * Preferred over `sessionDir` — works with any storage backend. + * + * When provided, the full tool-result text is passed to this function; + * the agent receives a placeholder that references the canonical path + * `/workspace/memo/session/tool-results/{toolCallId}.txt`. + */ + writeToolResultArtifact?: (toolCallId: string, content: string) => Promise; + /** + * @deprecated Use `writeToolResultArtifact` instead. + * Session directory used to persist compacted tool results for later retrieval. + * When `writeToolResultArtifact` is also provided, it takes precedence. + */ sessionDir?: string; } @@ -43,6 +56,7 @@ export async function compactToolResults( maxTokensPerToolResult = 1500, keepRecentToolResults = 2, maxTokensPerRecentToolResult = Math.max(maxTokensPerToolResult * 4, 6000), + writeToolResultArtifact, sessionDir, } = options; @@ -76,14 +90,19 @@ export async function compactToolResults( const fullText = textBlocks.map((b) => b.text).join(""); const preview = fullText.slice(0, 200).replace(/\n+/g, " ").trim(); - const persistedPath = sessionDir + const canPersist = Boolean(writeToolResultArtifact ?? sessionDir); + const persistedPath = canPersist ? `/workspace/memo/session/tool-results/${trm.toolCallId}.txt` : null; - if (sessionDir && fullText.length > 0) { - const toolResultsDir = path.join(sessionDir, "tool-results"); - await mkdir(toolResultsDir, { recursive: true }); - await writeFile(path.join(toolResultsDir, `${trm.toolCallId}.txt`), fullText, "utf8"); + if (fullText.length > 0) { + if (writeToolResultArtifact) { + await writeToolResultArtifact(trm.toolCallId, fullText); + } else if (sessionDir) { + const toolResultsDir = path.join(sessionDir, "tool-results"); + await mkdir(toolResultsDir, { recursive: true }); + await writeFile(path.join(toolResultsDir, `${trm.toolCallId}.txt`), fullText, "utf8"); + } } const compactedText = diff --git a/packages/app/src/session/memo-fs.ts b/packages/app/src/session/memo-fs.ts new file mode 100644 index 0000000..5b0f446 --- /dev/null +++ b/packages/app/src/session/memo-fs.ts @@ -0,0 +1,101 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import type { MemoDocumentName, MemoDocumentScope } from "@agentrail/core"; + +/** + * The canonical root that the agent sandbox sees for memo resources. + * Paths below this root are the formal `/workspace/memo/**` contract. + */ +export const MEMO_FS_ROOT = "/workspace/memo" as const; + +/** + * A parsed representation of a `/workspace/memo/**` resource. + * + * - `document` — one of the three built-in memo documents (NOTES.md, TODO.md, USER.md) + * - `tool-result` — a compacted tool-result artifact + */ +export type MemoResource = + | { + kind: "document"; + scope: MemoDocumentScope; + name: MemoDocumentName; + } + | { + kind: "tool-result"; + toolCallId: string; + }; + +/** Returns `true` when `filePath` is inside `/workspace/memo/**`. */ +export function isMemoPath(filePath: string): boolean { + return filePath === MEMO_FS_ROOT || filePath.startsWith(MEMO_FS_ROOT + "/"); +} + +/** + * Parse a `/workspace/memo/**` path into a typed `MemoResource`. + * Throws when the path is not a valid memo path or names an unknown resource. + * + * @example + * parseMemoPath("/workspace/memo/session/NOTES.md") + * // → { kind: "document", scope: "session", name: "NOTES.md" } + * + * parseMemoPath("/workspace/memo/session/tool-results/abc123.txt") + * // → { kind: "tool-result", toolCallId: "abc123" } + */ +export function parseMemoPath(filePath: string): MemoResource { + if (!isMemoPath(filePath)) { + throw new Error(`[memo-fs] Not a memo path: "${filePath}". Must start with "${MEMO_FS_ROOT}".`); + } + + // Strip the root prefix and leading slash. + const relative = filePath.slice(MEMO_FS_ROOT.length + 1); // e.g. "session/NOTES.md" + const parts = relative.split("/"); + + if (parts.length < 2) { + throw new Error(`[memo-fs] Incomplete memo path: "${filePath}".`); + } + + const [rawScope, ...rest] = parts as [string, ...string[]]; + + if (rawScope === "session") { + if (rest[0] === "tool-results" && rest.length === 2) { + const toolCallId = rest[1]!.replace(/\.txt$/, ""); + return { kind: "tool-result", toolCallId }; + } + const name = rest[0]; + if (name === "NOTES.md" || name === "TODO.md") { + return { kind: "document", scope: "session", name }; + } + throw new Error(`[memo-fs] Unknown session memo document: "${name}" in path "${filePath}".`); + } + + if (rawScope === "user") { + const name = rest[0]; + if (name === "USER.md") { + return { kind: "document", scope: "user", name }; + } + throw new Error(`[memo-fs] Unknown user memo document: "${name}" in path "${filePath}".`); + } + + throw new Error(`[memo-fs] Unknown memo scope: "${rawScope}" in path "${filePath}".`); +} + +/** + * Compute the canonical `/workspace/memo/**` path for a memo document. + * + * @example + * memoDocumentPath("session", "NOTES.md") // → "/workspace/memo/session/NOTES.md" + * memoDocumentPath("user", "USER.md") // → "/workspace/memo/user/USER.md" + */ +export function memoDocumentPath(scope: MemoDocumentScope, name: MemoDocumentName): string { + return `${MEMO_FS_ROOT}/${scope}/${name}`; +} + +/** + * Compute the canonical `/workspace/memo/session/tool-results/{toolCallId}.txt` path. + */ +export function memoToolResultPath(toolCallId: string): string { + return `${MEMO_FS_ROOT}/session/tool-results/${toolCallId}.txt`; +} diff --git a/packages/app/src/session/session-manager.ts b/packages/app/src/session/session-manager.ts index 1ee729a..488ef5d 100644 --- a/packages/app/src/session/session-manager.ts +++ b/packages/app/src/session/session-manager.ts @@ -3,6 +3,11 @@ * Copyright (c) 2026 The Agentrail Authors */ +import { + buildCompactionMessage, + buildCompactionNotesEntry, + computeCompactionSplit, +} from "@/session/compaction-logic.js"; import { buildCompactionMetadata, buildMemoryEntry, @@ -372,13 +377,21 @@ export class SessionManager { const sessionDir = this.getSessionDir(tenantId, sessionId); const userDir = this.getUserDir(tenantId, userId); - const [notes, todo, user] = await Promise.all([ + const [notesRaw, todoRaw, userRaw] = await Promise.all([ buildMemoryEntry("NOTES.md", path.join(sessionDir, "NOTES.md")), buildMemoryEntry("TODO.md", path.join(sessionDir, "TODO.md")), buildMemoryEntry("USER.md", path.join(userDir, "USER.md")), ]); - return { sessionDir, userDir, entries: [notes, todo, user] }; + // Store canonical /workspace/memo/** paths in entries rather than host paths, + // so consumers never need to know the host filesystem layout. + return { + entries: [ + { ...notesRaw, path: "/workspace/memo/session/NOTES.md" }, + { ...todoRaw, path: "/workspace/memo/session/TODO.md" }, + { ...userRaw, path: "/workspace/memo/user/USER.md" }, + ], + }; } async readTodoFile(sessionRef: SessionRef): Promise { @@ -512,67 +525,32 @@ export class SessionManager { compressedCount: number; totalTokens: number; } | null> { - const { - triggerTokens = 60_000, - compactFraction = 1 / 3, - workspaceSnapshot, - force = false, - } = options; + const { compactFraction = 1 / 3, workspaceSnapshot, force = false } = options; const all = options.preloadedMessages ?? (await this.loadAllMessages(tenantId, sessionId)); - if (all.length < 6) return null; // too few messages to compact meaningfully - - const totalTokens = estimateMessageTokens(all); - if (!force && totalTokens <= triggerTokens) return null; - - // Split: compact oldest fraction, keep the rest. - // Advance the cut boundary past any toolResult messages to avoid orphaned - // tool results: a toolResult must always have its corresponding toolCall - // visible in the same context window. - let cutPoint = Math.max(1, Math.floor(all.length * compactFraction)); - while (cutPoint < all.length - 1 && all[cutPoint]!.role === "toolResult") { - cutPoint++; - } - // Edge case: first loop stopped at all.length-1 and it's still a toolResult - // (entire tail is toolResults). Pull back until toKeep starts on a safe boundary. - while (cutPoint > 1 && all[cutPoint]!.role === "toolResult") { - cutPoint--; - } - const toCompact = all.slice(0, cutPoint); - const toKeep = all.slice(cutPoint); + const split = computeCompactionSplit(all, { + triggerTokens: options.triggerTokens, + compactFraction, + force, + }); + if (!split) return null; + + const { toCompact, toKeep, totalTokens } = split; const summary = await summarizeFn(toCompact); const timestamp = new Date().toISOString(); const archiveId = await this.getNextCompactionArchiveId(tenantId, sessionId); - // Build the synthetic replacement message (UserMessage shape) - const compactionLines = [ - `[Conversation compacted at ${timestamp}. Full history preserved in messages.compactions/${archiveId}.jsonl.`, - `Archive ID: ${archiveId}`, - `${toCompact.length} messages (${Math.round( - totalTokens * compactFraction, - )} tokens estimated) were compressed.`, - ``, - `Summary of compressed conversation:`, + const compactionMessage = buildCompactionMessage({ summary, - ]; - - if (workspaceSnapshot && workspaceSnapshot.trim().length > 0) { - compactionLines.push( - ``, - `Sandbox workspace at time of compaction:`, - workspaceSnapshot.trim(), - ); - } - - compactionLines.push(`]`); - - const compactionMessage: Message = { - role: "user", - content: compactionLines.join("\n"), - timestamp: Date.now(), - } as Message; + archiveId, + toCompact, + totalTokens, + compactFraction, + timestamp, + workspaceSnapshot, + }); const sessionDir = this.getSessionDir(tenantId, sessionId); const messagesFile = path.join(sessionDir, "messages.jsonl"); @@ -590,15 +568,12 @@ export class SessionManager { await writeFile(messagesFile, newLines, "utf8"); // Append summary to NOTES.md so Memory Index picks it up on next request - const notesEntry = [ - ``, - `## Compaction Summary [${timestamp}]`, - ``, - `*(${toCompact.length} messages compressed, full history in \`messages.compactions/${archiveId}.jsonl\`)*`, - ``, + const notesEntry = buildCompactionNotesEntry({ summary, - ``, - ].join("\n"); + archiveId, + compressedCount: toCompact.length, + timestamp, + }); await appendFile(notesFile, notesEntry, "utf8"); return { @@ -675,4 +650,96 @@ export class SessionManager { return null; } } + + // ─── AgentrailSessionStore: Memo Documents ──────────────────────────────── + + async readMemoryDocument( + tenantId: string, + ownerId: string, + scope: "session" | "user", + name: "NOTES.md" | "TODO.md" | "USER.md", + ): Promise { + const filePath = this.getMemoDocumentPath(tenantId, ownerId, scope, name); + try { + return await readFile(filePath, "utf8"); + } catch { + return null; + } + } + + async writeMemoryDocument( + tenantId: string, + ownerId: string, + scope: "session" | "user", + name: "NOTES.md" | "TODO.md" | "USER.md", + content: string, + ): Promise { + const filePath = this.getMemoDocumentPath(tenantId, ownerId, scope, name); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, content, "utf8"); + } + + async appendMemoryDocument( + tenantId: string, + ownerId: string, + scope: "session" | "user", + name: "NOTES.md" | "TODO.md" | "USER.md", + content: string, + ): Promise { + const filePath = this.getMemoDocumentPath(tenantId, ownerId, scope, name); + await mkdir(path.dirname(filePath), { recursive: true }); + await appendFile(filePath, content, "utf8"); + } + + private getMemoDocumentPath( + tenantId: string, + ownerId: string, + scope: "session" | "user", + name: string, + ): string { + if (scope === "user") { + return path.join(this.getUserDir(tenantId, ownerId), name); + } + return path.join(this.getSessionDir(tenantId, ownerId), name); + } + + // ─── AgentrailSessionStore: Tool-Result Artifacts ───────────────────────── + + async readToolResultArtifact(sessionRef: SessionRef, toolCallId: string): Promise { + const { tenantId, sessionId } = this.resolveSessionRef(sessionRef); + const filePath = this.getToolResultArtifactPath(tenantId, sessionId, toolCallId); + try { + return await readFile(filePath, "utf8"); + } catch { + return null; + } + } + + async writeToolResultArtifact( + sessionRef: SessionRef, + toolCallId: string, + content: string, + ): Promise { + const { tenantId, sessionId } = this.resolveSessionRef(sessionRef); + const filePath = this.getToolResultArtifactPath(tenantId, sessionId, toolCallId); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, content, "utf8"); + } + + private getToolResultArtifactPath( + tenantId: string, + sessionId: string, + toolCallId: string, + ): string { + return path.join(this.getSessionDir(tenantId, sessionId), "tool-results", `${toolCallId}.txt`); + } + + // ─── UserSessionLister ──────────────────────────────────────────────────── + + async listSessionsByUser( + tenantId: string, + userId: string, + ): Promise<{ sessionId: string; updatedAt: number }[]> { + return this.listSessionIdsByUser(tenantId, userId); + } } diff --git a/packages/app/src/session/user-session-lister.ts b/packages/app/src/session/user-session-lister.ts new file mode 100644 index 0000000..ebe0558 --- /dev/null +++ b/packages/app/src/session/user-session-lister.ts @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import type { SessionMeta } from "@agentrail/core"; + +/** + * Provides user-scoped session listing for background services such as the + * user-memory consolidation plugin. + * + * Kept separate from `AgentrailSessionStore` because this is a read-side + * administrative capability, not a per-request store operation. + * + * `SessionManager` implements this interface via its existing + * `listSessionIdsByUser` method. + */ +export interface UserSessionLister { + listSessionsByUser(tenantId: string, userId: string): Promise; +} diff --git a/packages/app/test/reactive-compaction.test.ts b/packages/app/test/reactive-compaction.test.ts index 2a00257..d573d7f 100644 --- a/packages/app/test/reactive-compaction.test.ts +++ b/packages/app/test/reactive-compaction.test.ts @@ -8,7 +8,7 @@ import { createDefaultCapabilityTransformContext, } from "@agentrail/capabilities"; import type { AssistantMessage, MemoryIndex, Message, ToolResultMessage } from "@agentrail/core"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -203,9 +203,21 @@ describe("memory-context compaction integration", () => { buildMemoryIndex: async () => statefulMemoryIndex, listKnowledgeMetadatas: async () => [], listSkills: async () => [], - compactMessages: (messages: Message[], ctx?: { sessionDir?: string }) => + writeToolResultArtifact: async (toolCallId: string, content: string) => { + const toolResultsDir = path.join(sessionDir, "tool-results"); + await mkdir(toolResultsDir, { recursive: true }); + await writeFile(path.join(toolResultsDir, `${toolCallId}.txt`), content, "utf8"); + }, + compactMessages: ( + messages: Message[], + ctx?: { + writeToolResultArtifact?: (toolCallId: string, content: string) => Promise; + sessionDir?: string; + }, + ) => compactToolResults(messages, { keepRecentToolResults: 0, + writeToolResultArtifact: ctx?.writeToolResultArtifact, sessionDir: ctx?.sessionDir, }), }; @@ -213,7 +225,6 @@ describe("memory-context compaction integration", () => { const sharedState = { cachedContextMsgs: null, cacheExpiry: 0, - capturedSessionDir: undefined, }; const transform = createDefaultCapabilityTransformContext(options, sharedState); const providers = createDefaultCapabilityContextProviders(options, sharedState); diff --git a/packages/capabilities/src/index.ts b/packages/capabilities/src/index.ts index e272f86..d701dcb 100644 --- a/packages/capabilities/src/index.ts +++ b/packages/capabilities/src/index.ts @@ -119,6 +119,7 @@ export { buildSkillTool } from "@/skills/index.js"; export { OrchestrationManager, createFilesystemOrchestrationPersistence, + recoverOrchestrationState, } from "@/orchestration/index.js"; export type { AgentInputEnvelope, @@ -129,8 +130,11 @@ export type { ManagedAgentInstance, OrchestrationAgent, OrchestrationEvent, + OrchestrationMailboxEvent, OrchestrationMailboxState, OrchestrationPersistence, + OrchestrationSnapshot, + RecoveredOrchestrationState, StartRunInput, SubAgentRuntime, SubagentWorkerConfig, diff --git a/packages/capabilities/src/memory/context.ts b/packages/capabilities/src/memory/context.ts index 614e34d..2119f17 100644 --- a/packages/capabilities/src/memory/context.ts +++ b/packages/capabilities/src/memory/context.ts @@ -18,6 +18,7 @@ import type { ContextProvider, TransformContextFn, UserMessage } from "@agentrai export interface DefaultCapabilityContextState { cachedContextMsgs: UserMessage[] | null; cacheExpiry: number; + /** @deprecated No longer used; `writeToolResultArtifact` is passed through options. */ capturedSessionDir?: string; } @@ -25,7 +26,6 @@ export function createDefaultCapabilityContextState(): DefaultCapabilityContextS return { cachedContextMsgs: null, cacheExpiry: 0, - capturedSessionDir: undefined, }; } @@ -89,7 +89,6 @@ async function ensureCachedContextMessages( } } - state.capturedSessionDir = rawIndex.sessionDir; state.cachedContextMsgs = built; state.cacheExpiry = now + cacheTtlMs; @@ -107,7 +106,11 @@ export function createDefaultCapabilityTransformContext( return async (messages) => { await ensureCachedContextMessages(options, state); - return Promise.resolve(compactMessages(messages, { sessionDir: state.capturedSessionDir })); + return Promise.resolve( + compactMessages(messages, { + writeToolResultArtifact: options.writeToolResultArtifact, + }), + ); }; } diff --git a/packages/capabilities/src/memory/index.ts b/packages/capabilities/src/memory/index.ts index 7dfe08f..a501ab6 100644 --- a/packages/capabilities/src/memory/index.ts +++ b/packages/capabilities/src/memory/index.ts @@ -11,7 +11,7 @@ import { } from "@/memory/context.js"; import type { SkillMeta } from "@/skills/types.js"; import type { CapabilityBuildContext, CapabilityDescriptor } from "@/types.js"; -import type { MemoryIndex, Message } from "@agentrail/core"; +import type { MemoryIndex, Message, SessionRef } from "@agentrail/core"; export type { DefaultCapabilityContextOptions } from "@/memory/types.js"; @@ -20,6 +20,7 @@ export interface MemorySessionContext { tenantId: string; userId: string; sessionId: string; + sessionRef: SessionRef; } /** @@ -36,10 +37,48 @@ export interface MemoryContextBuilders { listSkills?(ctx: MemorySessionContext): Promise; /** Returns the current workspace snapshot from the sandbox, if available. */ listWorkspaceSnapshot?(ctx: MemorySessionContext): Promise; - /** Compacts message history to reduce context window usage. */ + /** + * Persists a compacted tool-result artifact to the backing store for the + * current session. Called by `compactMessages` (via `ctx.writeToolResultArtifact`) + * when a tool result is too large to keep inline. + * + * Implement this builder to unlock artifact persistence with any storage + * backend. The implementation should also call + * `sandboxManager.refreshMemoMirror(ctx.sessionId, ...)` when a live sandbox + * exists so the agent can immediately read the new artifact from inside the + * container. + * + * @example + * ```ts + * writeToolResultArtifact: async (ctx, toolCallId, content) => { + * const sessionRef = `${ctx.tenantId}:${ctx.sessionId}`; + * await Promise.all([ + * store.writeToolResultArtifact?.(sessionRef, toolCallId, content), + * sandboxManager.refreshMemoMirror( + * ctx.sessionId, + * `/workspace/memo/session/tool-results/${toolCallId}.txt`, + * content, + * ), + * ]); + * } + * ``` + */ + writeToolResultArtifact?( + ctx: MemorySessionContext, + toolCallId: string, + content: string, + ): Promise; + /** + * Compacts message history to reduce context window usage. + * Receives `ctx.writeToolResultArtifact` when `writeToolResultArtifact` is + * configured above — use it instead of the deprecated `sessionDir`. + */ compactMessages?( messages: Message[], - ctx?: { sessionDir?: string }, + ctx?: { + /** Persists a compacted tool-result artifact. Prefer over `sessionDir`. */ + writeToolResultArtifact?: (toolCallId: string, content: string) => Promise; + }, ): Message[] | Promise; /** When true, skills are delegated to a managed sub-agent. Defaults to false. */ delegateSkillsToSubAgent?: boolean; @@ -97,6 +136,7 @@ export function memoryContext( tenantId: ctx.tenantId, userId: ctx.userId, sessionId: ctx.sessionId, + sessionRef: ctx.sessionRef, }; const state = getState(ctx); return createDefaultCapabilityContextProviders( @@ -128,6 +168,7 @@ export function memoryContext( tenantId: ctx.tenantId, userId: ctx.userId, sessionId: ctx.sessionId, + sessionRef: ctx.sessionRef, }; const state = getState(ctx); @@ -149,6 +190,10 @@ export function memoryContext( listWorkspaceSnapshot: builders.listWorkspaceSnapshot ? () => builders.listWorkspaceSnapshot!(sessionCtx) : undefined, + writeToolResultArtifact: builders.writeToolResultArtifact + ? (toolCallId, content) => + builders.writeToolResultArtifact!(sessionCtx, toolCallId, content) + : undefined, compactMessages: builders.compactMessages, }, state, diff --git a/packages/capabilities/src/memory/messages.ts b/packages/capabilities/src/memory/messages.ts index ca5dc42..525dbd4 100644 --- a/packages/capabilities/src/memory/messages.ts +++ b/packages/capabilities/src/memory/messages.ts @@ -36,21 +36,36 @@ export function makeDateContextMessage(timestamp = Date.now()): UserMessage { }; } -/** Rewrites local memory paths into sandbox-visible paths for model context. */ +/** + * @deprecated This function is no longer required. + * + * `buildMemoryIndex` now produces canonical `/workspace/memo/**` paths directly. + * This function is kept as a passthrough for backward compatibility and will be + * removed in a future release. + */ export function translateMemoryPaths(index: MemoryIndex): MemoryIndex { + // Paths are already canonical (/workspace/memo/**), so no translation is needed. + // We still apply any legacy host-path rewriting for callers that have not yet + // migrated to the new buildMemoryIndex output. + const sessionDir = index.sessionDir; + const userDir = index.userDir; + + if (!sessionDir && !userDir) { + // New canonical index: return as-is. + return index; + } + const translatePath = (filePath: string): string => { - if (filePath.startsWith(index.sessionDir)) { - return "/workspace/memo/session" + filePath.slice(index.sessionDir.length); + if (sessionDir && filePath.startsWith(sessionDir)) { + return "/workspace/memo/session" + filePath.slice(sessionDir.length); } - if (filePath.startsWith(index.userDir)) { - return "/workspace/memo/user" + filePath.slice(index.userDir.length); + if (userDir && filePath.startsWith(userDir)) { + return "/workspace/memo/user" + filePath.slice(userDir.length); } return filePath; }; return { - sessionDir: "/workspace/memo/session", - userDir: "/workspace/memo/user", entries: index.entries.map((entry: MemoryIndexEntry) => ({ ...entry, path: translatePath(entry.path), @@ -60,12 +75,7 @@ export function translateMemoryPaths(index: MemoryIndex): MemoryIndex { /** Creates a synthetic message that summarizes session and user memory files. */ export function makeMemoryIndexMessage(index: MemoryIndex, timestamp = Date.now()): UserMessage { - const lines: string[] = [ - "[Memory Index]", - `Session dir : ${index.sessionDir}`, - `User dir : ${index.userDir}`, - "", - ]; + const lines: string[] = ["[Memory Index]", "Memo root: /workspace/memo", ""]; for (const entry of index.entries) { if (!entry.exists) { diff --git a/packages/capabilities/src/memory/types.ts b/packages/capabilities/src/memory/types.ts index f0776ab..ca37721 100644 --- a/packages/capabilities/src/memory/types.ts +++ b/packages/capabilities/src/memory/types.ts @@ -19,9 +19,19 @@ export interface DefaultCapabilityContextOptions { listKnowledgeMetadatas(): Promise<(KBMetadata | null)[]>; listSkills(): Promise; listWorkspaceSnapshot?(): Promise; + /** + * Callback to persist compacted tool-result artifacts. + * When provided, it is passed to `compactMessages` so the compaction + * layer can store artifacts via the active session store rather than + * writing directly to the filesystem. + */ + writeToolResultArtifact?: (toolCallId: string, content: string) => Promise; compactMessages?( messages: Message[], - ctx?: { sessionDir?: string }, + ctx?: { + /** Callback to persist a compacted tool-result artifact. */ + writeToolResultArtifact?: (toolCallId: string, content: string) => Promise; + }, ): Message[] | Promise; } diff --git a/packages/capabilities/src/orchestration/persistence.ts b/packages/capabilities/src/orchestration/persistence.ts index 038ae1e..fd2d6aa 100644 --- a/packages/capabilities/src/orchestration/persistence.ts +++ b/packages/capabilities/src/orchestration/persistence.ts @@ -25,6 +25,18 @@ export interface OrchestrationPersistence { loadMailboxEvents(agentId: string): Promise; loadMailboxState(agentId: string): Promise; writeMailboxState(agentId: string, state: OrchestrationMailboxState): Promise; + + /** + * Load the persisted message history for a managed sub-agent. + * Used by the worker process to resume a paused sub-agent across turn boundaries. + */ + loadAgentHistory(agentId: string): Promise; + + /** + * Persist the message history for a managed sub-agent after each turn. + * The history is keyed by `agentId` within the orchestration session. + */ + writeAgentHistory(agentId: string, history: unknown[]): Promise; } // Filesystem persistence still resolves a session root internally, but the @@ -51,5 +63,7 @@ export function createFilesystemOrchestrationPersistence( loadMailboxEvents: (agentId) => store.loadMailboxEvents(agentId), loadMailboxState: (agentId) => store.loadMailboxState(agentId), writeMailboxState: (agentId, state) => store.writeMailboxState(agentId, state), + loadAgentHistory: (agentId) => store.loadAgentHistory(agentId), + writeAgentHistory: (agentId, history) => store.writeAgentHistory(agentId, history), }; } diff --git a/packages/capabilities/src/orchestration/worker/subagent-worker.ts b/packages/capabilities/src/orchestration/worker/subagent-worker.ts index 85d7df7..11fce34 100644 --- a/packages/capabilities/src/orchestration/worker/subagent-worker.ts +++ b/packages/capabilities/src/orchestration/worker/subagent-worker.ts @@ -12,6 +12,7 @@ import { type OrchestrationMailboxState, } from "@/orchestration/index.js"; import { createFilesystemOrchestrationStore } from "@/orchestration/orchestration-store.js"; +import type { OrchestrationMailboxEvent } from "@/orchestration/types.js"; import { type SubAgentRuntime, type SubagentWorkerConfig, @@ -89,7 +90,11 @@ async function handleMessage(message: WorkerMessage): Promise { } async function handleInit(message: WorkerInitMessage): Promise { - workerStore = createWorkerStore(message.dataDir, message.sessionRef); + workerStore = await createWorkerStoreFromConfig( + message.storageConfig, + message.dataDir, + message.sessionRef, + ); state = { tenantId: message.tenantId, userId: message.userId, @@ -471,6 +476,79 @@ function createWorkerStore(dataDir: string, sessionRef: WorkerInitMessage["sessi }; } +/** + * Creates a worker store from the serialised `WorkerStorageConfig`. + * + * For `type: "filesystem"` (or when no config is provided) this delegates to + * the existing synchronous `createWorkerStore` helper. + * + * For `type: "postgres"` the worker dynamically imports + * `@agentrail/storage-postgres` at runtime to avoid introducing a hard compile- + * time dependency from `@agentrail/capabilities` onto the optional postgres + * package (which itself depends on `@agentrail/capabilities`). + */ +async function createWorkerStoreFromConfig( + config: WorkerInitMessage["storageConfig"], + fallbackDataDir: string, + sessionRef: WorkerInitMessage["sessionRef"], +): Promise> { + if (config?.type !== "postgres") { + const dataDir = config?.type === "filesystem" ? config.dataDir : fallbackDataDir; + return createWorkerStore(dataDir, sessionRef); + } + + // Dynamic import: avoids a circular hard-dependency while still supporting + // postgres orchestration in worker processes when the package is installed. + let pgMod: { + createSqlClient: (opts: { connectionString: string; schema?: string }) => unknown; + PostgresOrchestrationPersistence: new ( + sql: unknown, + sessionRef: unknown, + schema?: string, + ) => { + loadMailboxState(agentId: string): Promise; + loadMailboxEvents(agentId: string): Promise; + writeMailboxState(agentId: string, state: OrchestrationMailboxState): Promise; + loadAgentHistory(agentId: string): Promise; + writeAgentHistory(agentId: string, history: unknown[]): Promise; + }; + }; + + try { + // Use Function constructor so tsc does not statically analyse the import + // and complain about the optional peer dependency. + pgMod = (await new Function("p", "return import(p)")( + "@agentrail/storage-postgres", + )) as typeof pgMod; + } catch { + throw new Error( + `Sub-agent worker: storageConfig.type is "postgres" but ` + + `@agentrail/storage-postgres is not installed. ` + + `Install it with: npm install @agentrail/storage-postgres`, + ); + } + + const sql = pgMod.createSqlClient({ + connectionString: config.connectionString, + ...(config.schema ? { schema: config.schema } : {}), + }); + const persistence = new pgMod.PostgresOrchestrationPersistence(sql, sessionRef, config.schema); + + return { + loadMailboxState: (agentId: string) => persistence.loadMailboxState(agentId), + loadMailboxEvents: (agentId: string) => + persistence.loadMailboxEvents(agentId) as Promise, + writeMailboxState: (agentId: string, state: OrchestrationMailboxState) => + persistence.writeMailboxState(agentId, state), + async loadHistory(agentId: string): Promise { + const history = await persistence.loadAgentHistory(agentId); + return history as Message[]; + }, + writeHistory: (agentId: string, history: Message[]) => + persistence.writeAgentHistory(agentId, history), + }; +} + function send(message: ParentMessage): void { if (typeof process.send === "function") { process.send(message); diff --git a/packages/capabilities/src/orchestration/worker/worker-messages.ts b/packages/capabilities/src/orchestration/worker/worker-messages.ts index 025881b..868c23a 100644 --- a/packages/capabilities/src/orchestration/worker/worker-messages.ts +++ b/packages/capabilities/src/orchestration/worker/worker-messages.ts @@ -10,6 +10,16 @@ export interface WorkerConfigMessage { fakeExecution?: "" | "echo"; } +/** + * Serialisable storage configuration sent from the orchestration manager to + * each sub-agent worker process. The worker uses this to reconstruct the + * correct `OrchestrationPersistence` without coupling to the parent's + * runtime context. + */ +export type WorkerStorageConfig = + | { type: "filesystem"; dataDir: string } + | { type: "postgres"; connectionString: string; schema?: string }; + /** Parent-to-worker initialization payload sent once after fork. */ export interface WorkerInitMessage { type: "init"; @@ -17,7 +27,18 @@ export interface WorkerInitMessage { userId: string; sessionId: string; sessionRef: SessionRef; + /** + * @deprecated Use `storageConfig` instead. Kept for backward compatibility + * while all callers migrate to `storageConfig`. Will be required when + * `storageConfig` is absent. + */ dataDir: string; + /** + * Serialisable storage configuration for the worker process. + * When present, the worker uses this to create the `OrchestrationPersistence` + * for the sub-agent session. Defaults to `{ type: "filesystem", dataDir }`. + */ + storageConfig?: WorkerStorageConfig; // biome-ignore lint/suspicious/noExplicitAny: Worker state is serialized runtimeConfig: any; workerConfig?: WorkerConfigMessage; diff --git a/packages/capabilities/src/sandbox/index.ts b/packages/capabilities/src/sandbox/index.ts index 4d59fb7..5f15159 100644 --- a/packages/capabilities/src/sandbox/index.ts +++ b/packages/capabilities/src/sandbox/index.ts @@ -11,6 +11,7 @@ export type { RunOptions, SandboxEntry, SandboxManagerOptions, + SandboxMemoProvider, } from "@/sandbox/sandbox-manager.js"; export { createSandboxedBash } from "@/sandbox/tools/sandboxed-bash.js"; diff --git a/packages/capabilities/src/sandbox/sandbox-manager.ts b/packages/capabilities/src/sandbox/sandbox-manager.ts index 31b9101..09dd76e 100644 --- a/packages/capabilities/src/sandbox/sandbox-manager.ts +++ b/packages/capabilities/src/sandbox/sandbox-manager.ts @@ -5,8 +5,9 @@ import Docker from "dockerode"; import { randomUUID } from "node:crypto"; -import { mkdir } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import * as net from "node:net"; +import * as os from "node:os"; import * as path from "node:path"; import { PassThrough } from "node:stream"; import tar from "tar-stream"; @@ -23,6 +24,74 @@ const MAX_OUTPUT_BYTES = 1024 * 1024; // 1 MB // ============================================================================ // ============================================================================ +/** + * Provider that gives `SandboxManager` bidirectional access to memo documents + * and tool-result artifacts stored in any backend (filesystem or database). + * + * When set on `SandboxManagerOptions.memoProvider`: + * - **Read path**: memo documents are snapshotted into a temporary host + * directory at sandbox creation time so that `/workspace/memo/**` paths are + * always populated inside the container. + * - **Write path**: writes made via the sandboxed `Write` / `Edit` tools to + * memo paths are propagated back to the underlying store so the backing + * store stays consistent with what the agent sees in the sandbox. + * + * Both `SessionManager` and `PostgresSessionStore` implement all methods of + * this interface and can be passed directly. + */ +export interface SandboxMemoProvider { + // ── Read ────────────────────────────────────────────────────────────────── + + /** Reads a named memo document. Returns `null` when absent. */ + readMemoryDocument( + tenantId: string, + ownerId: string, + scope: "session" | "user", + name: string, + ): Promise; + + /** + * Returns all tool-call IDs whose compacted artifacts are stored in this + * session. Used to pre-populate `/workspace/memo/session/tool-results/` + * inside the container at creation time. + * + * Optional — when absent, tool-result artifacts are not snapshotted and + * `/workspace/memo/session/tool-results/` will be empty inside the sandbox. + */ + listToolResultArtifactIds?(sessionRef: string): Promise; + + /** + * Reads a compacted tool-result artifact by tool call ID. + * Required when `listToolResultArtifactIds` is implemented. + */ + readToolResultArtifact?(sessionRef: string, toolCallId: string): Promise; + + // ── Write ───────────────────────────────────────────────────────────────── + + /** + * Persists a memo document back to the underlying store after the agent + * modifies it via the sandboxed `Write` or `Edit` tool. + * + * Optional — when absent, writes to memo paths only update the local temp + * directory and are not propagated to the backing store. + */ + writeMemoryDocument?( + tenantId: string, + ownerId: string, + scope: "session" | "user", + name: string, + content: string, + ): Promise; + + /** + * Persists a compacted tool-result artifact back to the underlying store + * after the agent writes to `/workspace/memo/session/tool-results/.txt`. + * + * Optional — when absent, such writes are not propagated to the store. + */ + writeToolResultArtifact?(sessionRef: string, toolCallId: string, content: string): Promise; +} + /** Active sandbox record for one session. */ export interface SandboxEntry { containerId: string; @@ -30,6 +99,15 @@ export interface SandboxEntry { workspaceDir: string; memoSessionDir: string; memoUserDir: string; + /** Stored so write-back methods can call the correct store APIs. */ + tenantId: string; + userId: string; + sessionId: string; + /** + * When a `memoProvider` is used, memo docs are snapshotted here before + * container creation so they can be bind-mounted. Cleaned up on destroy. + */ + tempMemoBase?: string; } /** Execution options for one `docker exec` call. */ @@ -61,6 +139,25 @@ export interface SandboxManagerOptions { image?: string; idleTimeoutMs?: number; docker?: Docker; + /** + * Provider used to snapshot memo documents into the sandbox at creation time. + * + * When set, the manager fetches session and user memo files (`NOTES.md`, + * `TODO.md`, `USER.md`) from the provider before starting the container and + * writes them to a temporary host directory that is bind-mounted at + * `/workspace/memo/session` and `/workspace/memo/user` inside the container. + * + * This ensures `/workspace/memo/**` paths are populated even when the host + * uses a database-backed session store that does not write files to disk. + * + * The snapshot is taken once at sandbox creation. Updates to memo documents + * while the sandbox is running are not automatically reflected (the container + * must be restarted to pick up changes). + * + * Pass your session store directly — both `SessionManager` and + * `PostgresSessionStore` implement the required `readMemoryDocument` method. + */ + memoProvider?: SandboxMemoProvider; } // ============================================================================ @@ -107,6 +204,25 @@ async function waitForHealth(url: string, timeoutMs: number): Promise { /** * Manages one Docker-backed isolated sandbox per session. * + * ## Memo path semantics (`/workspace/memo/**`) + * + * The manager bind-mounts two directories into every container: + * + * - `/workspace/memo/session` — session-scoped memo files (`NOTES.md`, `TODO.md`) + * - `/workspace/memo/user` — user-scoped memo files (`USER.md`) + * + * **Filesystem backend** (`SessionManager` / default): the real + * `/tenants/...` directories are mounted directly. + * + * **Database/custom backend**: pass a `memoProvider` in `SandboxManagerOptions`. + * The manager will snapshot memo documents from the provider into a temporary + * host directory before starting the container and bind-mount that directory. + * The snapshot is taken once at sandbox creation; updates while the container + * is running are not automatically reflected. + * + * Without a `memoProvider`, `/workspace/memo/**` will be empty for non-filesystem + * backends. + * * @see {@link https://agentrail.run/guides/use-capability-packages} */ export class SandboxManager { @@ -116,6 +232,7 @@ export class SandboxManager { private readonly idleTimers = new Map>(); private readonly image: string; private readonly idleTimeoutMs: number; + private readonly memoProvider: SandboxMemoProvider | undefined; constructor( private readonly dataDir: string, @@ -124,6 +241,7 @@ export class SandboxManager { this.docker = options.docker ?? new Docker(); this.image = options.image ?? SANDBOX_IMAGE; this.idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS; + this.memoProvider = options.memoProvider; } async ensureSandbox(sessionId: string, tenantId: string, userId: string): Promise { @@ -157,14 +275,80 @@ export class SandboxManager { userId: string, ): Promise { const workspaceDir = path.join(this.dataDir, "sandboxes", sessionId); - const memoSessionDir = path.join(this.dataDir, "tenants", tenantId, "sessions", sessionId); - const memoUserDir = path.join(this.dataDir, "tenants", tenantId, "users", userId); const skillsDir = path.join(this.dataDir, "skills"); + // ── Memo directories ──────────────────────────────────────────────────── + // + // When a memoProvider is configured (database-backed stores), snapshot memo + // documents to a temporary host directory before container creation so they + // can be bind-mounted. The temp dir is stored in the SandboxEntry and + // cleaned up on destroySandbox(). + // + // Without a memoProvider the legacy behaviour is preserved: the real + // dataDir/tenants/... directories are bind-mounted directly. + let memoSessionDir: string; + let memoUserDir: string; + let tempMemoBase: string | undefined; + + if (this.memoProvider) { + tempMemoBase = await mkdtemp(path.join(os.tmpdir(), `agentrail-memo-${sessionId}-`)); + memoSessionDir = path.join(tempMemoBase, "session"); + memoUserDir = path.join(tempMemoBase, "user"); + const toolResultsDir = path.join(memoSessionDir, "tool-results"); + await Promise.all([ + mkdir(memoSessionDir, { recursive: true }), + mkdir(memoUserDir, { recursive: true }), + mkdir(toolResultsDir, { recursive: true }), + ]); + + // Build the session reference used for artifact lookups. + // Follows the same convention as resolveSessionRef in other modules. + const sessionRef = `${tenantId}:${sessionId}`; + + // Snapshot known memo files and tool-result artifacts in parallel. + const snapshotTasks: Promise[] = [ + this._snapshotMemoDoc( + this.memoProvider, + tenantId, + sessionId, + "session", + "NOTES.md", + memoSessionDir, + ), + this._snapshotMemoDoc( + this.memoProvider, + tenantId, + sessionId, + "session", + "TODO.md", + memoSessionDir, + ), + this._snapshotMemoDoc(this.memoProvider, tenantId, userId, "user", "USER.md", memoUserDir), + ]; + + if (this.memoProvider.listToolResultArtifactIds && this.memoProvider.readToolResultArtifact) { + const { listToolResultArtifactIds, readToolResultArtifact } = this.memoProvider; + snapshotTasks.push( + this._snapshotToolResultArtifacts( + { listToolResultArtifactIds, readToolResultArtifact }, + sessionRef, + toolResultsDir, + ), + ); + } + + await Promise.all(snapshotTasks); + } else { + memoSessionDir = path.join(this.dataDir, "tenants", tenantId, "sessions", sessionId); + memoUserDir = path.join(this.dataDir, "tenants", tenantId, "users", userId); + await Promise.all([ + mkdir(memoSessionDir, { recursive: true }), + mkdir(memoUserDir, { recursive: true }), + ]); + } + await Promise.all([ mkdir(workspaceDir, { recursive: true }), - mkdir(memoSessionDir, { recursive: true }), - mkdir(memoUserDir, { recursive: true }), mkdir(skillsDir, { recursive: true }), ]); @@ -186,8 +370,8 @@ export class SandboxManager { CpuQuota: 100_000, Binds: [ `${workspaceDir}:/workspace`, - `${memoSessionDir}:/workspace/memo/session`, - `${memoUserDir}:/workspace/memo/user`, + `${memoSessionDir}:/workspace/memo/session:ro`, + `${memoUserDir}:/workspace/memo/user:ro`, `${skillsDir}:/skills:ro`, ], PortBindings: { @@ -207,7 +391,67 @@ export class SandboxManager { `[sandbox] Container ready for session ${sessionId} (browser port: ${browserPort})`, ); - return { containerId: container.id, browserPort, workspaceDir, memoSessionDir, memoUserDir }; + return { + containerId: container.id, + browserPort, + workspaceDir, + memoSessionDir, + memoUserDir, + tenantId, + userId, + sessionId, + ...(tempMemoBase ? { tempMemoBase } : {}), + }; + } + + /** Fetches one memo document from the provider and writes it to `targetDir`. */ + private async _snapshotMemoDoc( + provider: SandboxMemoProvider, + tenantId: string, + ownerId: string, + scope: "session" | "user", + name: string, + targetDir: string, + ): Promise { + try { + const content = await provider.readMemoryDocument(tenantId, ownerId, scope, name); + if (content !== null && content !== "") { + await writeFile(path.join(targetDir, name), content, "utf8"); + } + } catch { + // Non-fatal: if the store fails to read a memo doc, the sandbox starts + // without it rather than blocking sandbox creation entirely. + } + } + + /** + * Fetches all tool-result artifacts for `sessionRef` from the provider and + * writes each one to `toolResultsDir/.txt`. + */ + private async _snapshotToolResultArtifacts( + provider: Required< + Pick + >, + sessionRef: string, + toolResultsDir: string, + ): Promise { + try { + const ids = await provider.listToolResultArtifactIds(sessionRef); + await Promise.all( + ids.map(async (id) => { + try { + const content = await provider.readToolResultArtifact(sessionRef, id); + if (content !== null) { + await writeFile(path.join(toolResultsDir, `${id}.txt`), content, "utf8"); + } + } catch { + // Non-fatal: skip individual artifacts that fail to load. + } + }), + ); + } catch { + // Non-fatal: if listing fails the sandbox starts without pre-populated artifacts. + } } async runInSandbox(sessionId: string, cmd: string[], opts: RunOptions = {}): Promise { @@ -464,6 +708,15 @@ export class SandboxManager { ]).catch(() => undefined); } + /** + * Translates a container-side path into an absolute host filesystem path. + * + * Handles `/workspace/memo/session`, `/workspace/memo/user`, and `/workspace`. + * Returns an empty string for unrecognised paths. + * + * For database-backed stores with a `memoProvider`, memo paths resolve into + * the temporary snapshot directory created at sandbox creation time. + */ translateToHostPath(sessionId: string, containerPath: string): string { const entry = this.sandboxes.get(sessionId); if (!entry) throw new Error(`No sandbox found for session '${sessionId}'`); @@ -503,6 +756,107 @@ export class SandboxManager { return containerPath === "/tmp" || containerPath.startsWith("/tmp/"); } + /** + * Propagates a memo-path write back to the underlying store via + * `memoProvider`. Called by the sandboxed `Write` and `Edit` tools after + * they successfully update the local temp-mirror file. + * + * No-op when no `memoProvider` is configured or when the container path is + * not under `/workspace/memo/`. + */ + async writeMemoBack(sessionId: string, containerPath: string, content: string): Promise { + if (!this.memoProvider) return; + + const entry = this.sandboxes.get(sessionId); + if (!entry) return; + + const memoSessionPrefix = "/workspace/memo/session/"; + const memoUserPrefix = "/workspace/memo/user/"; + const toolResultsPrefix = `${memoSessionPrefix}tool-results/`; + + if (containerPath.startsWith(toolResultsPrefix)) { + // e.g. /workspace/memo/session/tool-results/.txt + if (!this.memoProvider.writeToolResultArtifact) return; + const filename = containerPath.slice(toolResultsPrefix.length); + const toolCallId = filename.endsWith(".txt") ? filename.slice(0, -4) : filename; + const sessionRef = `${entry.tenantId}:${entry.sessionId}`; + await this.memoProvider.writeToolResultArtifact(sessionRef, toolCallId, content); + } else if (containerPath.startsWith(memoSessionPrefix)) { + // e.g. /workspace/memo/session/NOTES.md + if (!this.memoProvider.writeMemoryDocument) return; + const name = containerPath.slice(memoSessionPrefix.length); + await this.memoProvider.writeMemoryDocument( + entry.tenantId, + entry.sessionId, + "session", + name, + content, + ); + } else if (containerPath.startsWith(memoUserPrefix)) { + // e.g. /workspace/memo/user/USER.md + if (!this.memoProvider.writeMemoryDocument) return; + const name = containerPath.slice(memoUserPrefix.length); + await this.memoProvider.writeMemoryDocument( + entry.tenantId, + entry.userId, + "user", + name, + content, + ); + } + } + + /** + * Updates a single file in the live host-side memo mirror without writing + * back to the store. Use this after a host-side write (e.g. compaction + * persisting a tool-result artifact) so the agent can immediately read the + * updated file from inside the container. + * + * No-op when the session has no active temp mirror (i.e. uses a filesystem + * backend where `memoSessionDir` / `memoUserDir` are already the real paths). + */ + async refreshMemoMirror( + sessionId: string, + containerPath: string, + content: string, + ): Promise { + const entry = this.sandboxes.get(sessionId); + if (!entry || !entry.tempMemoBase) return; + try { + const hostPath = this.translateToHostPath(sessionId, containerPath); + await mkdir(path.dirname(hostPath), { recursive: true }); + await writeFile(hostPath, content, "utf-8"); + } catch { + // Non-fatal: mirror refresh failure must not disrupt host-side logic. + } + } + + /** + * Refreshes the user-level memo file (`/workspace/memo/user/`) in the + * live mirror for **all** active sandboxes belonging to the given tenant+user + * pair. + * + * Call this after any host-side write to a user-scoped memo document (e.g. + * after `UserMemoryConsolidationService` rewrites `USER.md`) so that every + * concurrently running session sees the updated content immediately. + */ + async refreshUserMemoMirrorForAllSessions( + tenantId: string, + userId: string, + name: string, + content: string, + ): Promise { + const containerPath = `/workspace/memo/user/${name}`; + await Promise.allSettled( + [...this.sandboxes.entries()] + .filter( + ([, entry]) => + entry.tenantId === tenantId && entry.userId === userId && entry.tempMemoBase, + ) + .map(([sessionId]) => this.refreshMemoMirror(sessionId, containerPath, content)), + ); + } + async readFileInContainer(sessionId: string, containerPath: string): Promise { const result = await this.runInSandbox(sessionId, ["cat", containerPath]); if (result.exitCode !== 0) { @@ -597,6 +951,7 @@ export class SandboxManager { this.idleTimers.delete(sessionId); } + const entry = this.sandboxes.get(sessionId); this.sandboxes.delete(sessionId); this.pending.delete(sessionId); @@ -606,6 +961,11 @@ export class SandboxManager { } catch { // Container may not exist } + + // Clean up temp memo snapshot directory if one was created. + if (entry?.tempMemoBase) { + await rm(entry.tempMemoBase, { recursive: true, force: true }).catch(() => {}); + } } async destroyAll(): Promise { diff --git a/packages/capabilities/src/sandbox/tools/sandboxed-bash.ts b/packages/capabilities/src/sandbox/tools/sandboxed-bash.ts index 758a7cd..5dcde31 100644 --- a/packages/capabilities/src/sandbox/tools/sandboxed-bash.ts +++ b/packages/capabilities/src/sandbox/tools/sandboxed-bash.ts @@ -23,8 +23,8 @@ Usage: - Do NOT use this tool for file search — use the Grep tool instead. - Do NOT use this tool for file read/write — use the Read, Write, or Edit tools instead. - All file paths inside the sandbox start with /workspace. -- Session memory files (NOTES.md, TODO.md) are at /workspace/memo/session/. -- User profile (USER.md) is at /workspace/memo/user/USER.md.`; +- Session memory files (NOTES.md, TODO.md) are at /workspace/memo/session/; user profile (USER.md) is at /workspace/memo/user/USER.md. +- /workspace/memo/** is READ-ONLY for Bash — shell writes to memo paths will fail. Use the Write or Edit tools to persist memo changes.`; const parametersSchema = Type.Object({ command: Type.String({ description: "The shell command to execute inside the sandbox." }), diff --git a/packages/capabilities/src/sandbox/tools/sandboxed-edit.ts b/packages/capabilities/src/sandbox/tools/sandboxed-edit.ts index 4f3ce16..9e2f610 100644 --- a/packages/capabilities/src/sandbox/tools/sandboxed-edit.ts +++ b/packages/capabilities/src/sandbox/tools/sandboxed-edit.ts @@ -111,6 +111,7 @@ export function createSandboxedEdit( } else { const hostPath = manager.translateToHostPath(sessionId, file_path); await writeFile(hostPath, updated, "utf-8"); + await manager.writeMemoBack(sessionId, file_path, updated); } } catch (err) { const message = err instanceof Error ? err.message : String(err); diff --git a/packages/capabilities/src/sandbox/tools/sandboxed-write.ts b/packages/capabilities/src/sandbox/tools/sandboxed-write.ts index 865fbde..62daadf 100644 --- a/packages/capabilities/src/sandbox/tools/sandboxed-write.ts +++ b/packages/capabilities/src/sandbox/tools/sandboxed-write.ts @@ -55,6 +55,7 @@ export function createSandboxedWrite( const hostPath = manager.translateToHostPath(sessionId, file_path); await mkdir(dirname(hostPath), { recursive: true }); await writeFile(hostPath, contents, "utf-8"); + await manager.writeMemoBack(sessionId, file_path, contents); } } catch (err) { const message = err instanceof Error ? err.message : String(err); diff --git a/packages/capabilities/test/memory-context-compaction.test.ts b/packages/capabilities/test/memory-context-compaction.test.ts index bdacf5c..4eaec60 100644 --- a/packages/capabilities/test/memory-context-compaction.test.ts +++ b/packages/capabilities/test/memory-context-compaction.test.ts @@ -12,21 +12,27 @@ function userMsg(text: string): Message { } describe("createDefaultCapabilityTransformContext", () => { - it("passes sessionDir to compactMessages and awaits async results", async () => { - const sessionDir = "/tmp/agentrail-session"; + it("passes writeToolResultArtifact to compactMessages and awaits async results", async () => { const input = [userMsg("hello")]; - const memoryIndex: MemoryIndex = { - sessionDir, - userDir: "/tmp/agentrail-user", - entries: [], - }; + const memoryIndex: MemoryIndex = { entries: [] }; const compacted = [userMsg("compacted")]; - const compactMessages = vi.fn(async (messages: Message[], ctx?: { sessionDir?: string }) => { - await Promise.resolve(); - expect(messages).toEqual(input); - expect(ctx).toEqual({ sessionDir }); - return compacted; - }); + + const writeToolResultArtifact = vi.fn(async (_id: string, _content: string) => {}); + + const compactMessages = vi.fn( + async ( + messages: Message[], + ctx?: { + writeToolResultArtifact?: (toolCallId: string, content: string) => Promise; + sessionDir?: string; + }, + ) => { + await Promise.resolve(); + expect(messages).toEqual(input); + expect(ctx).toEqual({ writeToolResultArtifact }); + return compacted; + }, + ); const transform = createDefaultCapabilityTransformContext({ tenantId: "tenant-1", @@ -36,13 +42,14 @@ describe("createDefaultCapabilityTransformContext", () => { buildMemoryIndex: async () => memoryIndex, listKnowledgeMetadatas: async () => [], listSkills: async () => [], + writeToolResultArtifact, compactMessages, }); const result = await transform(input); expect(compactMessages).toHaveBeenCalledTimes(1); - expect(compactMessages).toHaveBeenCalledWith(input, { sessionDir }); + expect(compactMessages).toHaveBeenCalledWith(input, { writeToolResultArtifact }); expect(result.at(-1)).toEqual(compacted[0]); }); }); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e22d4d1..ad503e1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -50,6 +50,8 @@ export type { AgentrailSessionStore, ContextProvider, ContextProviderContext, + MemoDocumentName, + MemoDocumentScope, } from "@/session/contracts.js"; export { createSessionRef, resolveSessionRef } from "@/session/session-ref.js"; export type { SessionRef, SessionRefInfo } from "@/session/session-ref.js"; diff --git a/packages/core/src/session/contracts.ts b/packages/core/src/session/contracts.ts index 5eadbe1..7fad435 100644 --- a/packages/core/src/session/contracts.ts +++ b/packages/core/src/session/contracts.ts @@ -8,6 +8,18 @@ import type { TodoStorage } from "@/session/todo-storage.js"; import type { Message } from "@/types/message.types.js"; import type { Usage } from "@/types/usage.types.js"; +/** + * Scope of a memo document — `"session"` for per-session resources, + * `"user"` for cross-session user-level resources. + */ +export type MemoDocumentScope = "session" | "user"; + +/** + * Canonical names for the built-in memo documents that map to + * `/workspace/memo/{scope}/{name}` inside the agent sandbox. + */ +export type MemoDocumentName = "NOTES.md" | "TODO.md" | "USER.md"; + /** * Minimal storage surface the host runtime needs to load, persist, and compact * session history. The default file-backed `SessionManager` satisfies this shape, @@ -19,6 +31,8 @@ import type { Usage } from "@/types/usage.types.js"; * append new messages after each agent turn. * - **Usage tracking** — record per-turn token usage for billing / analytics. * - **Compaction** — optionally summarise old messages when the context window fills up. + * - **Memo documents** — optional hooks for `/workspace/memo/**` resources (NOTES, TODO, USER). + * - **Tool-result artifacts** — optional persistence for compacted tool-result text. * - **Extensions** — optional hooks for TODO storage and skill sub-agent logging. * * @see {@link https://agentrail.run/reference/session-store} @@ -116,6 +130,68 @@ export interface AgentrailSessionStore { finishedAt: number; }, ): Promise; + + // ─── Memo Documents (/workspace/memo/**) ────────────────────────────────── + + /** + * Read the full text of a memo document. + * + * - `scope = "session"`, `ownerId = sessionId` → `/workspace/memo/session/{name}` + * - `scope = "user"`, `ownerId = userId` → `/workspace/memo/user/{name}` + * + * Optional — when absent the runtime falls back to reading the corresponding + * file on the host filesystem. + */ + readMemoryDocument?( + tenantId: string, + ownerId: string, + scope: MemoDocumentScope, + name: MemoDocumentName, + ): Promise; + + /** + * Overwrite the full text of a memo document. + * Optional — when absent the runtime falls back to writing the host file. + */ + writeMemoryDocument?( + tenantId: string, + ownerId: string, + scope: MemoDocumentScope, + name: MemoDocumentName, + content: string, + ): Promise; + + /** + * Append text to a memo document (e.g. appending a compaction summary to NOTES.md). + * Optional — when absent the runtime falls back to appending to the host file. + */ + appendMemoryDocument?( + tenantId: string, + ownerId: string, + scope: MemoDocumentScope, + name: MemoDocumentName, + content: string, + ): Promise; + + // ─── Tool-Result Artifacts ───────────────────────────────────────────────── + + /** + * Read a compacted tool-result artifact by its tool-call ID. + * Artifacts are exposed to the agent at + * `/workspace/memo/session/tool-results/{toolCallId}.txt`. + * Optional — when absent the runtime falls back to reading the host file. + */ + readToolResultArtifact?(sessionRef: SessionRef, toolCallId: string): Promise; + + /** + * Persist a compacted tool-result artifact. + * Optional — when absent the runtime falls back to writing the host file. + */ + writeToolResultArtifact?( + sessionRef: SessionRef, + toolCallId: string, + content: string, + ): Promise; } /** Minimal context passed to context providers during message assembly. */ diff --git a/packages/core/src/session/types.ts b/packages/core/src/session/types.ts index 8f3da6e..69257c9 100644 --- a/packages/core/src/session/types.ts +++ b/packages/core/src/session/types.ts @@ -87,10 +87,18 @@ export interface MemoryIndexEntry { /** Session and user memory files exposed to hosted profiles and tools. */ export interface MemoryIndex { - /** Absolute path to the session directory */ - sessionDir: string; - /** Absolute path to the user directory (USER.md lives here) */ - userDir: string; + /** + * @deprecated No longer part of the public contract. + * Paths in `entries` are now always canonical `/workspace/memo/**` paths. + * Will be removed in a future release. + */ + sessionDir?: string; + /** + * @deprecated No longer part of the public contract. + * Paths in `entries` are now always canonical `/workspace/memo/**` paths. + * Will be removed in a future release. + */ + userDir?: string; entries: MemoryIndexEntry[]; } diff --git a/packages/storage-postgres/package.json b/packages/storage-postgres/package.json new file mode 100644 index 0000000..a2e8e32 --- /dev/null +++ b/packages/storage-postgres/package.json @@ -0,0 +1,49 @@ +{ + "name": "@agentrail/storage-postgres", + "version": "0.1.0", + "description": "PostgreSQL storage backend for Agentrail session, trace, orchestration, and inspector data.", + "type": "module", + "files": [ + "dist", + "README.md" + ], + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc && tsc-alias", + "dev": "tsc --watch", + "typecheck": "tsc --noEmit", + "clean": "rm -rf dist", + "test": "vitest run" + }, + "dependencies": { + "@agentrail/app": "workspace:*", + "@agentrail/capabilities": "workspace:*", + "@agentrail/core": "workspace:*", + "postgres": "^3.4.5" + }, + "devDependencies": { + "@testcontainers/postgresql": "^11.14.0", + "@types/node": "^22.0.0", + "testcontainers": "^11.14.0", + "tsc-alias": "^1.8.16", + "vite-tsconfig-paths": "^6.1.1", + "vitest": "^2.0.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/yai-dev/agentrail.git", + "directory": "packages/storage-postgres" + }, + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org/" + }, + "license": "Apache-2.0" +} diff --git a/packages/storage-postgres/src/client.ts b/packages/storage-postgres/src/client.ts new file mode 100644 index 0000000..ef5d60e --- /dev/null +++ b/packages/storage-postgres/src/client.ts @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import postgres from "postgres"; + +export type Sql = ReturnType; + +/** Options for creating a PostgreSQL-backed Agentrail storage backend. */ +export interface PostgresStorageOptions { + /** + * PostgreSQL connection string, e.g. + * `"postgres://user:password@localhost:5432/dbname"`. + */ + connectionString: string; + /** + * PostgreSQL schema name (default: `"agentrail"`). + * The schema must already exist; see `buildSchemaDDL()` to create it. + */ + schema?: string; + /** + * Maximum number of connections in the pool (default: 10). + */ + max?: number; +} + +/** + * Creates a `postgres` SQL client from the provided options. + * All Agentrail PostgreSQL backends share a single client instance. + */ +export function createSqlClient(options: PostgresStorageOptions): Sql { + return postgres(options.connectionString, { + max: options.max ?? 10, + idle_timeout: 30, + connect_timeout: 10, + }); +} + +/** + * Wraps an arbitrary value for safe JSONB storage via postgres.js. + * This cast is intentional: postgres.js accepts any serializable value at + * runtime but the TypeScript types require a strict `JSONValue` shape that + * domain objects like `Message` and `OrchestrationEvent` do not satisfy. + * + * Accepts both `Sql` and `TransactionSql` (which lack the full `Sql` surface). + */ +export function jsonParam( + sql: { json(value: unknown): ReturnType }, + value: unknown, +): ReturnType { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (sql as any).json(value); +} diff --git a/packages/storage-postgres/src/index.ts b/packages/storage-postgres/src/index.ts new file mode 100644 index 0000000..3eacf2b --- /dev/null +++ b/packages/storage-postgres/src/index.ts @@ -0,0 +1,14 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +export { createSqlClient, type PostgresStorageOptions, type Sql } from "./client.js"; +export { PostgresInspectorDataSource } from "./inspector-data-source.js"; +export { + PostgresOrchestrationPersistence, + createPostgresOrchestrationPersistence, +} from "./orchestration-persistence.js"; +export { buildSchemaDDL } from "./schema.js"; +export { PostgresSessionStore } from "./session-store.js"; +export { PostgresSessionTraceStore, createPostgresSessionTraceStore } from "./trace-store.js"; diff --git a/packages/storage-postgres/src/inspector-data-source.ts b/packages/storage-postgres/src/inspector-data-source.ts new file mode 100644 index 0000000..064369b --- /dev/null +++ b/packages/storage-postgres/src/inspector-data-source.ts @@ -0,0 +1,146 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import type { + InspectorDataSource, + InspectorSessionItem, + WorkflowTraceEventEnvelope, +} from "@agentrail/app"; +import type { + OrchestrationEvent, + OrchestrationSnapshot, + RecoveredOrchestrationState, +} from "@agentrail/capabilities"; +import { recoverOrchestrationState } from "@agentrail/capabilities"; +import type { Message } from "@agentrail/core"; +import type { Sql } from "./client.js"; + +/** + * PostgreSQL-backed `InspectorDataSource`. + * + * Reads session metadata, messages, trace envelopes, and orchestration state + * directly from PostgreSQL tables created by `buildSchemaDDL()`. + * Does not depend on any filesystem layout. + */ +export class PostgresInspectorDataSource implements InspectorDataSource { + constructor( + private readonly sql: Sql, + private readonly schema = "agentrail", + ) {} + + private get s() { + return this.schema; + } + + async listSessions(): Promise { + const rows = await this.sql< + { + tenant_id: string; + session_id: string; + user_id: string; + updated_at: string; + }[] + >` + SELECT tenant_id, session_id, user_id, updated_at + FROM ${this.sql(this.s)}.sessions + ORDER BY updated_at DESC + `; + + const items: InspectorSessionItem[] = await Promise.all( + rows.map(async (row) => { + const [turnsRows, traceRows] = await Promise.all([ + this.sql<{ count: string; total_tokens: string }[]>` + SELECT + COUNT(*) AS count, + COALESCE(SUM(input_tokens + output_tokens), 0) AS total_tokens + FROM ${this.sql(this.s)}.session_turns + WHERE tenant_id = ${row.tenant_id} AND session_id = ${row.session_id} + `, + this.sql<{ ts: string }[]>` + SELECT envelope->>'timestamp' AS ts + FROM ${this.sql(this.s)}.trace_envelopes + WHERE tenant_id = ${row.tenant_id} AND session_id = ${row.session_id} + ORDER BY seq DESC + LIMIT 1 + `, + ]); + + const turns = parseInt(turnsRows[0]?.count ?? "0", 10); + const tokens = parseInt(turnsRows[0]?.total_tokens ?? "0", 10); + const lastActive = traceRows[0]?.ts ?? new Date(Number(row.updated_at)).toISOString(); + + return { + tenantId: row.tenant_id, + sessionId: row.session_id, + userId: row.user_id, + lastActive, + turns: turns > 0 ? turns : undefined, + tokens: tokens > 0 ? tokens : undefined, + status: "idle" as const, + }; + }), + ); + + return items; + } + + async loadMessages(tenantId: string, sessionId: string): Promise { + const rows = await this.sql<{ message: Message }[]>` + SELECT message + FROM ${this.sql(this.s)}.session_messages + WHERE tenant_id = ${tenantId} AND session_id = ${sessionId} + ORDER BY seq ASC + `; + return rows.map((r) => r.message); + } + + async loadTraceEnvelopes( + tenantId: string, + sessionId: string, + ): Promise { + const rows = await this.sql<{ envelope: WorkflowTraceEventEnvelope }[]>` + SELECT envelope + FROM ${this.sql(this.s)}.trace_envelopes + WHERE tenant_id = ${tenantId} AND session_id = ${sessionId} + ORDER BY seq ASC + `; + return rows.map((r) => r.envelope); + } + + async loadOrchestrationState( + tenantId: string, + sessionId: string, + ): Promise { + const [snapshotRows, eventsRows] = await Promise.all([ + this.sql<{ snapshot: OrchestrationSnapshot }[]>` + SELECT snapshot + FROM ${this.sql(this.s)}.orchestration_snapshots + WHERE tenant_id = ${tenantId} AND session_id = ${sessionId} + `, + this.sql<{ event: OrchestrationEvent }[]>` + SELECT event + FROM ${this.sql(this.s)}.orchestration_events + WHERE tenant_id = ${tenantId} AND session_id = ${sessionId} + ORDER BY seq ASC + `, + ]); + + const snapshot = snapshotRows[0]?.snapshot ?? null; + const events = eventsRows.map((r) => r.event); + + if (!snapshot && events.length === 0) return null; + return recoverOrchestrationState(snapshot, events); + } + + async loadOrchestrationEvents(tenantId: string, sessionId: string): Promise { + const rows = await this.sql<{ event: unknown }[]>` + SELECT event + FROM ${this.sql(this.s)}.orchestration_events + WHERE tenant_id = ${tenantId} AND session_id = ${sessionId} + ORDER BY seq ASC + `; + return rows.map((r) => r.event); + } +} diff --git a/packages/storage-postgres/src/orchestration-persistence.ts b/packages/storage-postgres/src/orchestration-persistence.ts new file mode 100644 index 0000000..90bcdfe --- /dev/null +++ b/packages/storage-postgres/src/orchestration-persistence.ts @@ -0,0 +1,167 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import type { + OrchestrationEvent, + OrchestrationMailboxEvent, + OrchestrationMailboxState, + OrchestrationPersistence, + OrchestrationSnapshot, + RecoveredOrchestrationState, +} from "@agentrail/capabilities"; +import { recoverOrchestrationState } from "@agentrail/capabilities"; +import type { SessionRef } from "@agentrail/core"; +import { resolveSessionRef } from "@agentrail/core"; +import { jsonParam, type Sql } from "./client.js"; + +/** + * PostgreSQL-backed `OrchestrationPersistence`. + * + * Events, snapshots, mailbox state, and agent histories are all stored in + * PostgreSQL. Use `createPostgresOrchestrationPersistence` as the + * `createPersistence` option for `createOrchestrationRegistry`. + */ +export class PostgresOrchestrationPersistence implements OrchestrationPersistence { + private readonly tenantId: string; + private readonly sessionId: string; + + constructor( + private readonly sql: Sql, + sessionRef: SessionRef, + private readonly schema = "agentrail", + ) { + const resolved = resolveSessionRef(sessionRef); + this.tenantId = resolved.tenantId; + this.sessionId = resolved.sessionId; + } + + private get s() { + return this.schema; + } + + async appendEvent(event: OrchestrationEvent): Promise { + await this.sql` + INSERT INTO ${this.sql(this.s)}.orchestration_events + (tenant_id, session_id, event) + VALUES + (${this.tenantId}, ${this.sessionId}, ${jsonParam(this.sql, event)}) + `; + } + + async loadEvents(): Promise { + const rows = await this.sql<{ event: OrchestrationEvent }[]>` + SELECT event FROM ${this.sql(this.s)}.orchestration_events + WHERE tenant_id = ${this.tenantId} AND session_id = ${this.sessionId} + ORDER BY seq ASC + `; + return rows.map((r) => r.event); + } + + async loadSnapshot(): Promise { + const rows = await this.sql<{ snapshot: OrchestrationSnapshot }[]>` + SELECT snapshot FROM ${this.sql(this.s)}.orchestration_snapshots + WHERE tenant_id = ${this.tenantId} AND session_id = ${this.sessionId} + `; + return rows[0]?.snapshot ?? null; + } + + async writeCheckpoint(snapshot: OrchestrationSnapshot): Promise { + const now = Date.now(); + await this.sql` + INSERT INTO ${this.sql(this.s)}.orchestration_snapshots + (tenant_id, session_id, snapshot, updated_at) + VALUES + (${this.tenantId}, ${this.sessionId}, ${jsonParam(this.sql, snapshot)}, ${now}) + ON CONFLICT (tenant_id, session_id) + DO UPDATE SET snapshot = EXCLUDED.snapshot, updated_at = EXCLUDED.updated_at + `; + } + + async recoverState(): Promise { + const [snapshot, events] = await Promise.all([this.loadSnapshot(), this.loadEvents()]); + return recoverOrchestrationState(snapshot, events); + } + + async appendMailboxEvent(agentId: string, event: OrchestrationMailboxEvent): Promise { + await this.sql` + INSERT INTO ${this.sql(this.s)}.orchestration_mailbox_events + (tenant_id, session_id, agent_id, event) + VALUES + (${this.tenantId}, ${this.sessionId}, ${agentId}, ${jsonParam(this.sql, event)}) + `; + } + + async loadMailboxEvents(agentId: string): Promise { + const rows = await this.sql<{ event: OrchestrationMailboxEvent }[]>` + SELECT event FROM ${this.sql(this.s)}.orchestration_mailbox_events + WHERE tenant_id = ${this.tenantId} + AND session_id = ${this.sessionId} + AND agent_id = ${agentId} + ORDER BY seq ASC + `; + return rows.map((r) => r.event); + } + + async loadMailboxState(agentId: string): Promise { + const rows = await this.sql<{ state: OrchestrationMailboxState }[]>` + SELECT state FROM ${this.sql(this.s)}.orchestration_mailbox_states + WHERE tenant_id = ${this.tenantId} + AND session_id = ${this.sessionId} + AND agent_id = ${agentId} + `; + return ( + rows[0]?.state ?? { + processedEventCount: 0, + closeRequested: false, + pendingInputIds: [], + } + ); + } + + async writeMailboxState(agentId: string, state: OrchestrationMailboxState): Promise { + const now = Date.now(); + await this.sql` + INSERT INTO ${this.sql(this.s)}.orchestration_mailbox_states + (tenant_id, session_id, agent_id, state, updated_at) + VALUES + (${this.tenantId}, ${this.sessionId}, ${agentId}, ${jsonParam(this.sql, state)}, ${now}) + ON CONFLICT (tenant_id, session_id, agent_id) + DO UPDATE SET state = EXCLUDED.state, updated_at = EXCLUDED.updated_at + `; + } + + async loadAgentHistory(agentId: string): Promise { + const rows = await this.sql<{ history: unknown[] }[]>` + SELECT history FROM ${this.sql(this.s)}.orchestration_agent_histories + WHERE tenant_id = ${this.tenantId} + AND session_id = ${this.sessionId} + AND agent_id = ${agentId} + `; + return rows[0]?.history ?? []; + } + + async writeAgentHistory(agentId: string, history: unknown[]): Promise { + const now = Date.now(); + await this.sql` + INSERT INTO ${this.sql(this.s)}.orchestration_agent_histories + (tenant_id, session_id, agent_id, history, updated_at) + VALUES + (${this.tenantId}, ${this.sessionId}, ${agentId}, ${jsonParam(this.sql, history)}, ${now}) + ON CONFLICT (tenant_id, session_id, agent_id) + DO UPDATE SET history = EXCLUDED.history, updated_at = EXCLUDED.updated_at + `; + } +} + +/** + * Factory that creates a `PostgresOrchestrationPersistence` for each session. + * Pass the returned function as `createPersistence` to `createOrchestrationRegistry`. + */ +export function createPostgresOrchestrationPersistence( + sql: Sql, + schema = "agentrail", +): (sessionRef: SessionRef) => OrchestrationPersistence { + return (sessionRef) => new PostgresOrchestrationPersistence(sql, sessionRef, schema); +} diff --git a/packages/storage-postgres/src/schema.ts b/packages/storage-postgres/src/schema.ts new file mode 100644 index 0000000..01b9eb4 --- /dev/null +++ b/packages/storage-postgres/src/schema.ts @@ -0,0 +1,185 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + * + * PostgreSQL schema DDL for all Agentrail storage tables. + * + * All tables are created inside the schema specified at construction time + * (default: `agentrail`). Create the schema before running migrations: + * + * CREATE SCHEMA IF NOT EXISTS agentrail; + */ + +/** + * Returns the DDL statements that create the Agentrail storage schema. + * Pass the result to `sql.unsafe(ddl)` or execute it via `psql`. + * + * All statements are idempotent (uses `CREATE TABLE IF NOT EXISTS`, etc.). + * + * @param schema - PostgreSQL schema name (default: `"agentrail"`) + */ +export function buildSchemaDDL(schema = "agentrail"): string { + const s = schema; + return ` +-- ═══════════════════════════════════════════════════════════════════════ +-- Session / memory +-- ═══════════════════════════════════════════════════════════════════════ + +CREATE TABLE IF NOT EXISTS ${s}.sessions ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + user_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + title TEXT, + created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + updated_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + PRIMARY KEY (tenant_id, session_id) +); + +CREATE TABLE IF NOT EXISTS ${s}.session_messages ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + seq BIGSERIAL, + message JSONB NOT NULL, + PRIMARY KEY (tenant_id, session_id, seq) +); + +CREATE INDEX IF NOT EXISTS ${s}_session_messages_seq + ON ${s}.session_messages (tenant_id, session_id, seq DESC); + +CREATE TABLE IF NOT EXISTS ${s}.session_message_archives ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + archive_id TEXT NOT NULL, + messages JSONB NOT NULL, + archived_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + PRIMARY KEY (tenant_id, session_id, archive_id) +); + +CREATE TABLE IF NOT EXISTS ${s}.session_turns ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + turn_index INT NOT NULL, + input_tokens INT NOT NULL DEFAULT 0, + output_tokens INT NOT NULL DEFAULT 0, + cache_read_tokens INT NOT NULL DEFAULT 0, + cache_write_tokens INT NOT NULL DEFAULT 0, + recorded_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + PRIMARY KEY (tenant_id, session_id, turn_index) +); + +-- Memo documents: NOTES.md (session), TODO.md (session), USER.md (user) +CREATE TABLE IF NOT EXISTS ${s}.memory_documents ( + tenant_id TEXT NOT NULL, + owner_scope TEXT NOT NULL CHECK (owner_scope IN ('session', 'user')), + owner_id TEXT NOT NULL, + name TEXT NOT NULL CHECK (name IN ('NOTES.md', 'TODO.md', 'USER.md')), + content TEXT NOT NULL DEFAULT '', + updated_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + PRIMARY KEY (tenant_id, owner_scope, owner_id, name) +); + +-- Compacted tool-result artifacts +CREATE TABLE IF NOT EXISTS ${s}.tool_result_artifacts ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + tool_call_id TEXT NOT NULL, + content TEXT NOT NULL, + updated_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + PRIMARY KEY (tenant_id, session_id, tool_call_id) +); + +CREATE TABLE IF NOT EXISTS ${s}.session_todos ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + content TEXT NOT NULL DEFAULT '', + updated_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + PRIMARY KEY (tenant_id, session_id) +); + +CREATE TABLE IF NOT EXISTS ${s}.skill_sub_agent_logs ( + id BIGSERIAL PRIMARY KEY, + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + skill_name TEXT NOT NULL, + task TEXT NOT NULL, + input TEXT NOT NULL, + system_prompt TEXT NOT NULL, + messages JSONB NOT NULL, + result_text TEXT NOT NULL, + started_at BIGINT NOT NULL, + finished_at BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS ${s}_skill_logs_session + ON ${s}.skill_sub_agent_logs (tenant_id, session_id); + +-- ═══════════════════════════════════════════════════════════════════════ +-- Trace +-- ═══════════════════════════════════════════════════════════════════════ + +CREATE TABLE IF NOT EXISTS ${s}.trace_envelopes ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + seq BIGSERIAL, + envelope JSONB NOT NULL, + PRIMARY KEY (tenant_id, session_id, seq) +); + +CREATE INDEX IF NOT EXISTS ${s}_trace_envelopes_session + ON ${s}.trace_envelopes (tenant_id, session_id, seq ASC); + +-- ═══════════════════════════════════════════════════════════════════════ +-- Orchestration +-- ═══════════════════════════════════════════════════════════════════════ + +CREATE TABLE IF NOT EXISTS ${s}.orchestration_events ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + seq BIGSERIAL, + event JSONB NOT NULL, + PRIMARY KEY (tenant_id, session_id, seq) +); + +CREATE INDEX IF NOT EXISTS ${s}_orchestration_events_session + ON ${s}.orchestration_events (tenant_id, session_id, seq ASC); + +CREATE TABLE IF NOT EXISTS ${s}.orchestration_snapshots ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + snapshot JSONB NOT NULL, + updated_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + PRIMARY KEY (tenant_id, session_id) +); + +CREATE TABLE IF NOT EXISTS ${s}.orchestration_mailbox_events ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + seq BIGSERIAL, + event JSONB NOT NULL, + PRIMARY KEY (tenant_id, session_id, agent_id, seq) +); + +CREATE INDEX IF NOT EXISTS ${s}_mailbox_events_agent + ON ${s}.orchestration_mailbox_events (tenant_id, session_id, agent_id, seq ASC); + +CREATE TABLE IF NOT EXISTS ${s}.orchestration_mailbox_states ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + state JSONB NOT NULL, + updated_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + PRIMARY KEY (tenant_id, session_id, agent_id) +); + +CREATE TABLE IF NOT EXISTS ${s}.orchestration_agent_histories ( + tenant_id TEXT NOT NULL, + session_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + history JSONB NOT NULL, + updated_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + PRIMARY KEY (tenant_id, session_id, agent_id) +); +`; +} diff --git a/packages/storage-postgres/src/session-store.ts b/packages/storage-postgres/src/session-store.ts new file mode 100644 index 0000000..e5feb28 --- /dev/null +++ b/packages/storage-postgres/src/session-store.ts @@ -0,0 +1,337 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import { + buildCompactionMessage, + buildCompactionNotesEntry, + computeCompactionSplit, +} from "@agentrail/app"; +import type { + AgentrailSessionStore, + MemoDocumentName, + MemoDocumentScope, + Message, + SessionMeta, + SessionRef, + Usage, +} from "@agentrail/core"; +import { createSessionRef, resolveSessionRef } from "@agentrail/core"; +import { randomUUID } from "node:crypto"; +import type { Sql } from "./client.js"; +import { jsonParam } from "./client.js"; + +/** Simple token estimator matching the one in @agentrail/app. */ +function estimateMessageTokens(messages: Message[]): number { + let tokens = 0; + for (const msg of messages) { + const text = JSON.stringify(msg); + for (const ch of text) { + tokens += ch.codePointAt(0)! > 0x7f ? 1 : 0.25; + } + } + return Math.ceil(tokens); +} + +/** + * PostgreSQL-backed `AgentrailSessionStore`. + * + * All data is stored in the `agentrail` schema (or the schema provided at + * construction time). Run `buildSchemaDDL()` once to create the tables. + */ +export class PostgresSessionStore implements AgentrailSessionStore { + constructor( + private readonly sql: Sql, + private readonly schema: string = "agentrail", + ) {} + + private get s() { + return this.schema; + } + + async ping(): Promise { + await this.sql`SELECT 1`; + } + + async getOrCreate( + tenantId: string, + userId: string, + agentId: string, + sessionId?: string, + ): Promise<{ sessionId: string; sessionRef: SessionRef }> { + const sid = sessionId ?? randomUUID(); + const now = Date.now(); + + await this.sql` + INSERT INTO ${this.sql(this.s)}.sessions + (tenant_id, session_id, user_id, agent_id, created_at, updated_at) + VALUES + (${tenantId}, ${sid}, ${userId}, ${agentId}, ${now}, ${now}) + ON CONFLICT (tenant_id, session_id) DO NOTHING + `; + + const sessionRef = createSessionRef(tenantId, sid); + return { sessionId: sid, sessionRef }; + } + + async loadMessages(tenantId: string, sessionId: string, limit?: number): Promise { + if (limit !== undefined) { + const rows = await this.sql<{ message: Message }[]>` + SELECT message FROM ${this.sql(this.s)}.session_messages + WHERE tenant_id = ${tenantId} AND session_id = ${sessionId} + ORDER BY seq DESC + LIMIT ${limit} + `; + return rows.map((r) => r.message).reverse(); + } + return this.loadAllMessages(tenantId, sessionId); + } + + async loadMessagesWithBudget( + tenantId: string, + sessionId: string, + tokenBudget = 40_000, + ): Promise { + const all = await this.loadAllMessages(tenantId, sessionId); + if (all.length === 0) return []; + + let tokens = 0; + const result: Message[] = []; + for (let i = all.length - 1; i >= 0; i--) { + const msgTokens = estimateMessageTokens([all[i]!]); + if (tokens + msgTokens > tokenBudget && result.length > 0) break; + tokens += msgTokens; + result.unshift(all[i]!); + } + return result; + } + + async loadAllMessages(tenantId: string, sessionId: string): Promise { + const rows = await this.sql<{ message: Message }[]>` + SELECT message FROM ${this.sql(this.s)}.session_messages + WHERE tenant_id = ${tenantId} AND session_id = ${sessionId} + ORDER BY seq ASC + `; + return rows.map((r) => r.message); + } + + async appendMessages(tenantId: string, sessionId: string, messages: Message[]): Promise { + if (messages.length === 0) return; + const now = Date.now(); + + for (const msg of messages) { + await this.sql` + INSERT INTO ${this.sql(this.s)}.session_messages (tenant_id, session_id, message) + VALUES (${tenantId}, ${sessionId}, ${jsonParam(this.sql, msg)}) + `; + } + + await this.sql` + UPDATE ${this.sql(this.s)}.sessions + SET updated_at = ${now} + WHERE tenant_id = ${tenantId} AND session_id = ${sessionId} + `; + } + + async recordTurn(tenantId: string, sessionId: string, usage: Usage): Promise { + const now = Date.now(); + + const countRows = await this.sql<{ count: string }[]>` + SELECT COUNT(*) as count + FROM ${this.sql(this.s)}.session_turns + WHERE tenant_id = ${tenantId} AND session_id = ${sessionId} + `; + const turnIndex = parseInt(countRows[0]?.count ?? "0", 10); + + await this.sql` + INSERT INTO ${this.sql(this.s)}.session_turns + (tenant_id, session_id, turn_index, input_tokens, output_tokens, + cache_read_tokens, cache_write_tokens, recorded_at) + VALUES + (${tenantId}, ${sessionId}, ${turnIndex}, + ${usage.inputTokens ?? 0}, ${usage.outputTokens ?? 0}, + ${usage.cacheReadTokens ?? 0}, ${usage.cacheWriteTokens ?? 0}, + ${now}) + `; + } + + async compactIfNeeded( + tenantId: string, + sessionId: string, + summarizeFn: (messages: Message[]) => Promise, + options: { + triggerTokens?: number; + compactFraction?: number; + preloadedMessages?: Message[]; + workspaceSnapshot?: string; + } = {}, + ): Promise { + const { compactFraction = 1 / 3, workspaceSnapshot } = options; + + const all = options.preloadedMessages ?? (await this.loadAllMessages(tenantId, sessionId)); + + const split = computeCompactionSplit(all, { + triggerTokens: options.triggerTokens, + compactFraction, + }); + if (!split) return false; + + const { toCompact, toKeep, totalTokens } = split; + + const summary = await summarizeFn(toCompact); + const timestamp = new Date().toISOString(); + const archiveId = randomUUID(); + + const compactionMsg = buildCompactionMessage({ + summary, + archiveId, + toCompact, + totalTokens, + compactFraction, + timestamp, + workspaceSnapshot, + }); + + // Archive the compacted messages. + await this.sql` + INSERT INTO ${this.sql(this.s)}.session_message_archives + (tenant_id, session_id, archive_id, messages) + VALUES + (${tenantId}, ${sessionId}, ${archiveId}, ${jsonParam(this.sql, toCompact)}) + `; + + // Rewrite visible messages: delete all and re-insert compaction + kept. + await this.sql.begin(async (sql) => { + await sql` + DELETE FROM ${sql(this.s)}.session_messages + WHERE tenant_id = ${tenantId} AND session_id = ${sessionId} + `; + const newMessages = [compactionMsg, ...toKeep]; + for (const msg of newMessages) { + await sql` + INSERT INTO ${sql(this.s)}.session_messages (tenant_id, session_id, message) + VALUES (${tenantId}, ${sessionId}, ${jsonParam(sql, msg)}) + `; + } + }); + + // Append compaction summary to NOTES.md. + const notesEntry = buildCompactionNotesEntry({ + summary, + archiveId, + compressedCount: toCompact.length, + timestamp, + }); + await this.appendMemoryDocument(tenantId, sessionId, "session", "NOTES.md", notesEntry); + + return true; + } + + // ─── Memo Documents ──────────────────────────────────────────────────────── + + async readMemoryDocument( + tenantId: string, + ownerId: string, + scope: MemoDocumentScope, + name: MemoDocumentName, + ): Promise { + const rows = await this.sql<{ content: string }[]>` + SELECT content FROM ${this.sql(this.s)}.memory_documents + WHERE tenant_id = ${tenantId} + AND owner_scope = ${scope} + AND owner_id = ${ownerId} + AND name = ${name} + `; + return rows[0]?.content ?? null; + } + + async writeMemoryDocument( + tenantId: string, + ownerId: string, + scope: MemoDocumentScope, + name: MemoDocumentName, + content: string, + ): Promise { + const now = Date.now(); + await this.sql` + INSERT INTO ${this.sql(this.s)}.memory_documents + (tenant_id, owner_scope, owner_id, name, content, updated_at) + VALUES + (${tenantId}, ${scope}, ${ownerId}, ${name}, ${content}, ${now}) + ON CONFLICT (tenant_id, owner_scope, owner_id, name) + DO UPDATE SET content = EXCLUDED.content, updated_at = EXCLUDED.updated_at + `; + } + + async appendMemoryDocument( + tenantId: string, + ownerId: string, + scope: MemoDocumentScope, + name: MemoDocumentName, + content: string, + ): Promise { + const now = Date.now(); + await this.sql` + INSERT INTO ${this.sql(this.s)}.memory_documents + (tenant_id, owner_scope, owner_id, name, content, updated_at) + VALUES + (${tenantId}, ${scope}, ${ownerId}, ${name}, ${content}, ${now}) + ON CONFLICT (tenant_id, owner_scope, owner_id, name) + DO UPDATE SET + content = ${this.sql(this.s)}.memory_documents.content || EXCLUDED.content, + updated_at = EXCLUDED.updated_at + `; + } + + // ─── Tool-Result Artifacts ───────────────────────────────────────────────── + + async readToolResultArtifact(sessionRef: SessionRef, toolCallId: string): Promise { + const { tenantId, sessionId } = resolveSessionRef(sessionRef); + const rows = await this.sql<{ content: string }[]>` + SELECT content FROM ${this.sql(this.s)}.tool_result_artifacts + WHERE tenant_id = ${tenantId} + AND session_id = ${sessionId} + AND tool_call_id = ${toolCallId} + `; + return rows[0]?.content ?? null; + } + + async writeToolResultArtifact( + sessionRef: SessionRef, + toolCallId: string, + content: string, + ): Promise { + const { tenantId, sessionId } = resolveSessionRef(sessionRef); + const now = Date.now(); + await this.sql` + INSERT INTO ${this.sql(this.s)}.tool_result_artifacts + (tenant_id, session_id, tool_call_id, content, updated_at) + VALUES + (${tenantId}, ${sessionId}, ${toolCallId}, ${content}, ${now}) + ON CONFLICT (tenant_id, session_id, tool_call_id) + DO UPDATE SET content = EXCLUDED.content, updated_at = EXCLUDED.updated_at + `; + } + + // ─── UserSessionLister ──────────────────────────────────────────────────── + + /** + * Lists all sessions belonging to the given user, ordered by most recently + * updated first. Satisfies the `UserSessionLister` interface so this store + * can be passed directly to `UserMemoryConsolidationService`. + */ + async listSessionsByUser(tenantId: string, userId: string): Promise { + const rows = await this.sql<{ session_id: string; updated_at: string }[]>` + SELECT session_id, updated_at + FROM ${this.sql(this.s)}.sessions + WHERE tenant_id = ${tenantId} + AND user_id = ${userId} + ORDER BY updated_at DESC + `; + return rows.map((r) => ({ + sessionId: r.session_id, + updatedAt: Number(r.updated_at), + })); + } +} diff --git a/packages/storage-postgres/src/trace-store.ts b/packages/storage-postgres/src/trace-store.ts new file mode 100644 index 0000000..3a4ed97 --- /dev/null +++ b/packages/storage-postgres/src/trace-store.ts @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import type { SessionTraceStore } from "@agentrail/app"; +import type { SessionRef } from "@agentrail/core"; +import { resolveSessionRef } from "@agentrail/core"; +import { jsonParam, type Sql } from "./client.js"; + +/** + * PostgreSQL-backed `SessionTraceStore`. + * + * Envelopes are appended in insertion order and loaded back in the same order. + */ +export class PostgresSessionTraceStore< + TEnvelope = Record, +> implements SessionTraceStore { + private readonly tenantId: string; + private readonly sessionId: string; + + constructor( + private readonly sql: Sql, + sessionRef: SessionRef, + private readonly schema = "agentrail", + ) { + const resolved = resolveSessionRef(sessionRef); + this.tenantId = resolved.tenantId; + this.sessionId = resolved.sessionId; + } + + async appendEnvelope(envelope: TEnvelope): Promise { + await this.sql` + INSERT INTO ${this.sql(this.schema)}.trace_envelopes + (tenant_id, session_id, envelope) + VALUES + (${this.tenantId}, ${this.sessionId}, ${jsonParam(this.sql, envelope)}) + `; + } + + async loadEnvelopes(): Promise { + const rows = await this.sql<{ envelope: TEnvelope }[]>` + SELECT envelope FROM ${this.sql(this.schema)}.trace_envelopes + WHERE tenant_id = ${this.tenantId} AND session_id = ${this.sessionId} + ORDER BY seq ASC + `; + return rows.map((r) => r.envelope); + } +} + +/** + * Factory function that creates a `PostgresSessionTraceStore` for a given session. + * Use this as the `traceStoreFactory` option in `createAgentApp`. + */ +export function createPostgresSessionTraceStore( + sql: Sql, + schema = "agentrail", +): (sessionRef: SessionRef) => SessionTraceStore { + return (sessionRef) => new PostgresSessionTraceStore(sql, sessionRef, schema); +} diff --git a/packages/storage-postgres/test/helpers.ts b/packages/storage-postgres/test/helpers.ts new file mode 100644 index 0000000..afb1140 --- /dev/null +++ b/packages/storage-postgres/test/helpers.ts @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import { PostgreSqlContainer, type StartedPostgreSqlContainer } from "@testcontainers/postgresql"; +import { afterAll, beforeAll } from "vitest"; +import { createSqlClient, type Sql } from "../src/client.js"; +import { buildSchemaDDL } from "../src/schema.js"; + +export interface PgFixture { + sql: Sql; +} + +/** + * Starts a PostgreSQL test container and applies the Agentrail schema. + * Call this in a `describe` block — it registers `beforeAll` / `afterAll` hooks. + */ +export function usePgContainer(): PgFixture { + let container: StartedPostgreSqlContainer; + const fixture: PgFixture = { sql: null! }; + + beforeAll(async () => { + container = await new PostgreSqlContainer("postgres:16-alpine").start(); + + fixture.sql = createSqlClient({ + connectionString: container.getConnectionUri(), + }); + + // Create schema and tables. + await fixture.sql.unsafe(`CREATE SCHEMA IF NOT EXISTS agentrail;`); + await fixture.sql.unsafe(buildSchemaDDL("agentrail")); + }); + + afterAll(async () => { + await fixture.sql?.end(); + await container?.stop(); + }); + + return fixture; +} + +/** Helper to build a simple user message. */ +export function userMsg(text: string) { + return { + role: "user" as const, + content: text, + timestamp: Date.now(), + }; +} + +/** Helper to build a simple assistant message. */ +export function assistantMsg(text: string) { + return { + role: "assistant" as const, + content: [{ type: "text" as const, text }], + stopReason: "stop" as const, + provider: "mock", + modelId: "mock", + usage: { + inputTokens: 10, + outputTokens: 5, + cacheReadTokens: 0, + cacheWriteTokens: 0, + totalTokens: 15, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + timestamp: Date.now(), + }; +} diff --git a/packages/storage-postgres/test/inspector-data-source.test.ts b/packages/storage-postgres/test/inspector-data-source.test.ts new file mode 100644 index 0000000..528a49c --- /dev/null +++ b/packages/storage-postgres/test/inspector-data-source.test.ts @@ -0,0 +1,129 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import { describe, expect, it } from "vitest"; +import { PostgresInspectorDataSource } from "../src/inspector-data-source.js"; +import { PostgresOrchestrationPersistence } from "../src/orchestration-persistence.js"; +import { PostgresSessionStore } from "../src/session-store.js"; +import { PostgresSessionTraceStore } from "../src/trace-store.js"; +import { assistantMsg, usePgContainer, userMsg } from "./helpers.js"; + +describe("PostgresInspectorDataSource", () => { + const pg = usePgContainer(); + + async function seedSession(tenantId: string, userId: string) { + const store = new PostgresSessionStore(pg.sql); + const { sessionId, sessionRef } = await store.getOrCreate(tenantId, userId, "agent"); + + // Append a couple of messages. + await store.appendMessages(tenantId, sessionId, [ + userMsg("hello"), + assistantMsg("hi"), + ] as never); + + // Record a turn. + await store.recordTurn(tenantId, sessionId, { + inputTokens: 10, + outputTokens: 5, + cacheReadTokens: 0, + cacheWriteTokens: 0, + totalTokens: 15, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }); + + // Append a trace envelope. + const traceStore = new PostgresSessionTraceStore(pg.sql, sessionRef); + await traceStore.appendEnvelope({ + id: "env-1", + timestamp: new Date().toISOString(), + sequence: 0, + source: "runtime", + event: { type: "turn_end" }, + }); + + return { sessionId, sessionRef }; + } + + it("listSessions returns seeded sessions", async () => { + const { sessionId } = await seedSession("insp-t1", "insp-u1"); + const ds = new PostgresInspectorDataSource(pg.sql); + + const sessions = await ds.listSessions(); + const found = sessions.find((s) => s.sessionId === sessionId); + expect(found).toBeDefined(); + expect(found!.tenantId).toBe("insp-t1"); + expect(found!.userId).toBe("insp-u1"); + }); + + it("listSessions returns turn count from session_turns", async () => { + const { sessionId } = await seedSession("insp-t2", "insp-u2"); + const ds = new PostgresInspectorDataSource(pg.sql); + + const sessions = await ds.listSessions(); + const found = sessions.find((s) => s.sessionId === sessionId); + expect(found!.turns).toBe(1); + }); + + it("loadMessages returns messages in order", async () => { + const { sessionId } = await seedSession("insp-t3", "insp-u3"); + const ds = new PostgresInspectorDataSource(pg.sql); + + const messages = await ds.loadMessages("insp-t3", sessionId); + expect(messages).toHaveLength(2); + expect((messages[0] as { content: string }).content).toBe("hello"); + }); + + it("loadMessages returns empty for unknown session", async () => { + const ds = new PostgresInspectorDataSource(pg.sql); + const msgs = await ds.loadMessages("insp-t3", "no-such-session"); + expect(msgs).toHaveLength(0); + }); + + it("loadTraceEnvelopes returns envelopes in order", async () => { + const { sessionId } = await seedSession("insp-t4", "insp-u4"); + const ds = new PostgresInspectorDataSource(pg.sql); + + const envelopes = await ds.loadTraceEnvelopes("insp-t4", sessionId); + expect(envelopes.length).toBeGreaterThanOrEqual(1); + expect((envelopes[0] as { id: string }).id).toBe("env-1"); + }); + + it("loadOrchestrationState returns null when no events or snapshot", async () => { + const ds = new PostgresInspectorDataSource(pg.sql); + const state = await ds.loadOrchestrationState("insp-t5", "no-orch-session"); + expect(state).toBeNull(); + }); + + it("loadOrchestrationState returns non-null when a snapshot is stored", async () => { + const { sessionId, sessionRef } = await seedSession("insp-t6", "insp-u6"); + const p = new PostgresOrchestrationPersistence(pg.sql, sessionRef); + + // Use recoverState to get a valid empty snapshot, then checkpoint it. + const emptyState = await p.recoverState(); + await p.writeCheckpoint(emptyState.snapshot); + + const ds = new PostgresInspectorDataSource(pg.sql); + const state = await ds.loadOrchestrationState("insp-t6", sessionId); + expect(state).not.toBeNull(); + expect(state!.snapshot).toBeDefined(); + }); + + it("loadOrchestrationEvents returns raw events", async () => { + const { sessionId, sessionRef } = await seedSession("insp-t7", "insp-u7"); + const p = new PostgresOrchestrationPersistence(pg.sql, sessionRef); + + await p.appendEvent({ + type: "run_started", + eventId: "ev-raw-1", + runId: "run-y", + occurredAt: new Date().toISOString(), + } as never); + + const ds = new PostgresInspectorDataSource(pg.sql); + const events = await ds.loadOrchestrationEvents("insp-t7", sessionId); + expect(events).toHaveLength(1); + expect((events[0] as { eventId: string }).eventId).toBe("ev-raw-1"); + }); +}); diff --git a/packages/storage-postgres/test/orchestration-persistence.test.ts b/packages/storage-postgres/test/orchestration-persistence.test.ts new file mode 100644 index 0000000..0ddbbe7 --- /dev/null +++ b/packages/storage-postgres/test/orchestration-persistence.test.ts @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import type { OrchestrationEvent } from "@agentrail/capabilities"; +import { createSessionRef } from "@agentrail/core"; +import { describe, expect, it } from "vitest"; +import { + PostgresOrchestrationPersistence, + createPostgresOrchestrationPersistence, +} from "../src/orchestration-persistence.js"; +import { usePgContainer } from "./helpers.js"; + +function makeSessionRef(id: string) { + return createSessionRef("t1", id); +} + +describe("PostgresOrchestrationPersistence", () => { + const pg = usePgContainer(); + + it("appendEvent / loadEvents round-trips in order", async () => { + const p = new PostgresOrchestrationPersistence(pg.sql, makeSessionRef("orch-1")); + + const e1: OrchestrationEvent = { + type: "run_started", + runId: "run-1", + occurredAt: new Date().toISOString(), + eventId: "ev-1", + } as unknown as OrchestrationEvent; + + const e2: OrchestrationEvent = { + type: "agent_spawned", + agentId: "ag-1", + runId: "run-1", + occurredAt: new Date().toISOString(), + eventId: "ev-2", + } as unknown as OrchestrationEvent; + + await p.appendEvent(e1); + await p.appendEvent(e2); + + const events = await p.loadEvents(); + expect(events).toHaveLength(2); + expect((events[0] as { eventId: string }).eventId).toBe("ev-1"); + expect((events[1] as { eventId: string }).eventId).toBe("ev-2"); + }); + + it("loadSnapshot returns null when no snapshot written", async () => { + const p = new PostgresOrchestrationPersistence(pg.sql, makeSessionRef("orch-snap-empty")); + expect(await p.loadSnapshot()).toBeNull(); + }); + + it("writeCheckpoint / loadSnapshot round-trips", async () => { + const p = new PostgresOrchestrationPersistence(pg.sql, makeSessionRef("orch-snap-1")); + const snap = { + agents: {}, + runs: {}, + version: 1, + } as unknown as import("@agentrail/capabilities").OrchestrationSnapshot; + + await p.writeCheckpoint(snap); + const loaded = await p.loadSnapshot(); + expect(loaded).toMatchObject({ version: 1 }); + + // Overwrite. + await p.writeCheckpoint({ + ...snap, + version: 2, + } as unknown as import("@agentrail/capabilities").OrchestrationSnapshot); + expect(((await p.loadSnapshot()) as { version: number }).version).toBe(2); + }); + + it("recoverState returns empty state when no data", async () => { + const p = new PostgresOrchestrationPersistence(pg.sql, makeSessionRef("orch-recover-empty")); + const state = await p.recoverState(); + expect(state).toBeDefined(); + expect(state.snapshot.agents).toBeDefined(); + }); + + it("mailbox events — append / load round-trips", async () => { + const p = new PostgresOrchestrationPersistence(pg.sql, makeSessionRef("orch-mailbox-1")); + + const event = { + type: "input_received", + agentId: "ag-x", + inputId: "inp-1", + occurredAt: new Date().toISOString(), + } as unknown as import("@agentrail/capabilities").OrchestrationMailboxEvent; + + await p.appendMailboxEvent("ag-x", event); + + const events = await p.loadMailboxEvents("ag-x"); + expect(events).toHaveLength(1); + expect((events[0] as { inputId: string }).inputId).toBe("inp-1"); + + // Different agent's mailbox is empty. + expect(await p.loadMailboxEvents("ag-other")).toHaveLength(0); + }); + + it("mailbox state — default and write/load round-trips", async () => { + const p = new PostgresOrchestrationPersistence(pg.sql, makeSessionRef("orch-mbox-state")); + + const defaultState = await p.loadMailboxState("ag-z"); + expect(defaultState.processedEventCount).toBe(0); + expect(defaultState.closeRequested).toBe(false); + + const newState = { processedEventCount: 3, closeRequested: true, pendingInputIds: ["i1"] }; + await p.writeMailboxState("ag-z", newState); + + const loaded = await p.loadMailboxState("ag-z"); + expect(loaded.processedEventCount).toBe(3); + expect(loaded.closeRequested).toBe(true); + expect(loaded.pendingInputIds).toEqual(["i1"]); + }); + + it("agent history — default empty and write/load round-trips", async () => { + const p = new PostgresOrchestrationPersistence(pg.sql, makeSessionRef("orch-history-1")); + + expect(await p.loadAgentHistory("ag-h")).toEqual([]); + + const history = [ + { role: "user", content: "msg1" }, + { role: "assistant", content: "msg2" }, + ]; + await p.writeAgentHistory("ag-h", history); + + const loaded = await p.loadAgentHistory("ag-h"); + expect(loaded).toHaveLength(2); + expect((loaded[0] as { content: string }).content).toBe("msg1"); + + // Overwrite. + await p.writeAgentHistory("ag-h", [history[0]!]); + expect(await p.loadAgentHistory("ag-h")).toHaveLength(1); + }); + + it("createPostgresOrchestrationPersistence factory produces a working store", async () => { + const factory = createPostgresOrchestrationPersistence(pg.sql); + const p = factory(makeSessionRef("orch-factory-1")); + + expect(await p.loadEvents()).toHaveLength(0); + expect(await p.recoverState()).toBeDefined(); + }); + + it("different sessions are isolated", async () => { + const pA = new PostgresOrchestrationPersistence(pg.sql, makeSessionRef("orch-iso-a")); + const pB = new PostgresOrchestrationPersistence(pg.sql, makeSessionRef("orch-iso-b")); + + await pA.appendEvent({ + type: "run_started", + eventId: "e-iso", + } as unknown as OrchestrationEvent); + + expect(await pA.loadEvents()).toHaveLength(1); + expect(await pB.loadEvents()).toHaveLength(0); + }); +}); diff --git a/packages/storage-postgres/test/session-store.test.ts b/packages/storage-postgres/test/session-store.test.ts new file mode 100644 index 0000000..f54527f --- /dev/null +++ b/packages/storage-postgres/test/session-store.test.ts @@ -0,0 +1,149 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import { describe, expect, it } from "vitest"; +import { PostgresSessionStore } from "../src/session-store.js"; +import { assistantMsg, usePgContainer, userMsg } from "./helpers.js"; + +describe("PostgresSessionStore", () => { + const pg = usePgContainer(); + + it("getOrCreate — creates and is idempotent", async () => { + const store = new PostgresSessionStore(pg.sql); + const { sessionId, sessionRef } = await store.getOrCreate("t1", "u1", "agent"); + expect(sessionId).toBeTruthy(); + expect(sessionRef).toBeTruthy(); + + // Re-creating with the same sessionId should not throw. + const { sessionId: same } = await store.getOrCreate("t1", "u1", "agent", sessionId); + expect(same).toBe(sessionId); + }); + + it("appendMessages / loadAllMessages round-trips", async () => { + const store = new PostgresSessionStore(pg.sql); + const { sessionId } = await store.getOrCreate("t1", "u2", "agent"); + + const msgs = [userMsg("hello"), assistantMsg("hi there")]; + await store.appendMessages("t1", sessionId, msgs as never); + + const loaded = await store.loadAllMessages("t1", sessionId); + expect(loaded).toHaveLength(2); + expect((loaded[0] as { content: string }).content).toBe("hello"); + expect((loaded[1] as { content: { text: string }[] }).content[0]!.text).toBe("hi there"); + }); + + it("loadMessages with limit returns latest N", async () => { + const store = new PostgresSessionStore(pg.sql); + const { sessionId } = await store.getOrCreate("t1", "u3", "agent"); + + await store.appendMessages("t1", sessionId, [ + userMsg("a"), + userMsg("b"), + userMsg("c"), + ] as never); + + const latest = await store.loadMessages("t1", sessionId, 2); + expect(latest).toHaveLength(2); + expect((latest[0] as { content: string }).content).toBe("b"); + }); + + it("loadMessagesWithBudget respects token budget", async () => { + const store = new PostgresSessionStore(pg.sql); + const { sessionId } = await store.getOrCreate("t1", "u4", "agent"); + + // Each message is ~3 tokens + await store.appendMessages("t1", sessionId, [ + userMsg("x"), + userMsg("y"), + userMsg("z"), + ] as never); + + // Very tight budget should return only the most recent messages. + const result = await store.loadMessagesWithBudget("t1", sessionId, 1); + expect(result.length).toBeGreaterThanOrEqual(1); + expect(result.length).toBeLessThanOrEqual(3); + }); + + it("recordTurn persists without error", async () => { + const store = new PostgresSessionStore(pg.sql); + const { sessionId } = await store.getOrCreate("t1", "u5", "agent"); + + await expect( + store.recordTurn("t1", sessionId, { + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 0, + cacheWriteTokens: 0, + totalTokens: 150, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }), + ).resolves.toBeUndefined(); + }); + + it("ping succeeds", async () => { + const store = new PostgresSessionStore(pg.sql); + await expect(store.ping()).resolves.toBeUndefined(); + }); + + // ─── Memory Documents ────────────────────────────────────────────────────── + + it("readMemoryDocument returns null when absent", async () => { + const store = new PostgresSessionStore(pg.sql); + const { sessionId } = await store.getOrCreate("t1", "memo-u1", "agent"); + const result = await store.readMemoryDocument("t1", sessionId, "session", "NOTES.md"); + expect(result).toBeNull(); + }); + + it("writeMemoryDocument / readMemoryDocument round-trips", async () => { + const store = new PostgresSessionStore(pg.sql); + const { sessionId } = await store.getOrCreate("t1", "memo-u2", "agent"); + + await store.writeMemoryDocument("t1", sessionId, "session", "NOTES.md", "initial content"); + const result = await store.readMemoryDocument("t1", sessionId, "session", "NOTES.md"); + expect(result).toBe("initial content"); + + // Overwrite. + await store.writeMemoryDocument("t1", sessionId, "session", "NOTES.md", "updated content"); + const updated = await store.readMemoryDocument("t1", sessionId, "session", "NOTES.md"); + expect(updated).toBe("updated content"); + }); + + it("appendMemoryDocument concatenates content", async () => { + const store = new PostgresSessionStore(pg.sql); + const { sessionId } = await store.getOrCreate("t1", "memo-u3", "agent"); + + await store.writeMemoryDocument("t1", sessionId, "session", "NOTES.md", "line1\n"); + await store.appendMemoryDocument("t1", sessionId, "session", "NOTES.md", "line2\n"); + const result = await store.readMemoryDocument("t1", sessionId, "session", "NOTES.md"); + expect(result).toBe("line1\nline2\n"); + }); + + it("USER.md scoped to user owner", async () => { + const store = new PostgresSessionStore(pg.sql); + await store.getOrCreate("t1", "usr-scope", "agent"); + + await store.writeMemoryDocument("t1", "usr-scope", "user", "USER.md", "user prefs"); + const result = await store.readMemoryDocument("t1", "usr-scope", "user", "USER.md"); + expect(result).toBe("user prefs"); + }); + + // ─── Tool-Result Artifacts ───────────────────────────────────────────────── + + it("writeToolResultArtifact / readToolResultArtifact round-trips", async () => { + const store = new PostgresSessionStore(pg.sql); + const { sessionId, sessionRef } = await store.getOrCreate("t1", "art-u1", "agent"); + + await store.writeToolResultArtifact(sessionRef, "tool-abc", "artifact content"); + const result = await store.readToolResultArtifact(sessionRef, "tool-abc"); + expect(result).toBe("artifact content"); + }); + + it("readToolResultArtifact returns null when absent", async () => { + const store = new PostgresSessionStore(pg.sql); + const { sessionRef } = await store.getOrCreate("t1", "art-u2", "agent"); + const result = await store.readToolResultArtifact(sessionRef, "missing-id"); + expect(result).toBeNull(); + }); +}); diff --git a/packages/storage-postgres/test/trace-store.test.ts b/packages/storage-postgres/test/trace-store.test.ts new file mode 100644 index 0000000..0fb32cb --- /dev/null +++ b/packages/storage-postgres/test/trace-store.test.ts @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 The Agentrail Authors + */ + +import { createSessionRef } from "@agentrail/core"; +import { describe, expect, it } from "vitest"; +import { PostgresSessionTraceStore, createPostgresSessionTraceStore } from "../src/trace-store.js"; +import { usePgContainer } from "./helpers.js"; + +describe("PostgresSessionTraceStore", () => { + const pg = usePgContainer(); + + it("appendEnvelope / loadEnvelopes round-trips in insertion order", async () => { + const sessionRef = createSessionRef("t1", "sess-trace-1"); + const store = new PostgresSessionTraceStore(pg.sql, sessionRef); + + const e1 = { + id: "e1", + timestamp: "2026-01-01T00:00:00Z", + sequence: 0, + source: "runtime", + event: { type: "turn_start" }, + }; + const e2 = { + id: "e2", + timestamp: "2026-01-01T00:00:01Z", + sequence: 1, + source: "runtime", + event: { type: "turn_end" }, + }; + + await store.appendEnvelope(e1); + await store.appendEnvelope(e2); + + const loaded = await store.loadEnvelopes(); + expect(loaded).toHaveLength(2); + expect(loaded[0]).toMatchObject({ id: "e1" }); + expect(loaded[1]).toMatchObject({ id: "e2" }); + }); + + it("loadEnvelopes returns empty array when no envelopes exist", async () => { + const sessionRef = createSessionRef("t1", "sess-trace-empty"); + const store = new PostgresSessionTraceStore(pg.sql, sessionRef); + const loaded = await store.loadEnvelopes(); + expect(loaded).toHaveLength(0); + }); + + it("createPostgresSessionTraceStore factory produces a working store", async () => { + const factory = createPostgresSessionTraceStore(pg.sql); + const sessionRef = createSessionRef("t1", "sess-trace-factory"); + const store = factory(sessionRef); + + await store.appendEnvelope({ + id: "factory-e1", + timestamp: new Date().toISOString(), + sequence: 0, + source: "runtime", + event: { type: "ping" }, + }); + const loaded = await store.loadEnvelopes(); + expect(loaded).toHaveLength(1); + }); + + it("different sessions are isolated", async () => { + const ref1 = createSessionRef("t1", "trace-iso-a"); + const ref2 = createSessionRef("t1", "trace-iso-b"); + + const storeA = new PostgresSessionTraceStore(pg.sql, ref1); + const storeB = new PostgresSessionTraceStore(pg.sql, ref2); + + await storeA.appendEnvelope({ + id: "a-only", + timestamp: new Date().toISOString(), + sequence: 0, + source: "runtime", + event: {}, + }); + + expect(await storeA.loadEnvelopes()).toHaveLength(1); + expect(await storeB.loadEnvelopes()).toHaveLength(0); + }); +}); diff --git a/packages/storage-postgres/tsconfig.json b/packages/storage-postgres/tsconfig.json new file mode 100644 index 0000000..bf8f92c --- /dev/null +++ b/packages/storage-postgres/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "composite": true, + "baseUrl": ".", + "paths": { "@/*": ["./src/*"] } + }, + "references": [{ "path": "../core" }, { "path": "../capabilities" }, { "path": "../app" }], + "include": ["src"] +} diff --git a/packages/storage-postgres/tsconfig.test.json b/packages/storage-postgres/tsconfig.test.json new file mode 100644 index 0000000..c3bfe83 --- /dev/null +++ b/packages/storage-postgres/tsconfig.test.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "noEmit": true, + "baseUrl": ".", + "paths": { + "@/*": ["./src/*"] + } + }, + "references": [{ "path": "../core" }, { "path": "../capabilities" }, { "path": "../app" }], + "include": ["src", "test"] +} diff --git a/packages/storage-postgres/vitest.config.ts b/packages/storage-postgres/vitest.config.ts new file mode 100644 index 0000000..eb32e32 --- /dev/null +++ b/packages/storage-postgres/vitest.config.ts @@ -0,0 +1,25 @@ +import { resolve } from "node:path"; +import tsconfigPaths from "vite-tsconfig-paths"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [tsconfigPaths()], + resolve: { + alias: { + "@agentrail/core/providers": resolve(__dirname, "../core/src/llm/providers/index.ts"), + "@agentrail/core": resolve(__dirname, "../core/src/index.ts"), + "@agentrail/capabilities": resolve(__dirname, "../capabilities/src/index.ts"), + "@agentrail/app": resolve(__dirname, "../app/src/index.ts"), + }, + }, + test: { + environment: "node", + include: ["test/**/*.test.ts"], + // testcontainers pulls a Docker image on first run; give it plenty of time. + testTimeout: 120_000, + hookTimeout: 120_000, + coverage: { + provider: "v8", + }, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cd6148a..4a930e1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -172,7 +172,7 @@ importers: version: 1.8.16 vite-tsconfig-paths: specifier: ^6.1.1 - version: 6.1.1(typescript@5.9.3)(vite@5.4.21(@types/node@22.19.15)) + version: 6.1.1(typescript@5.9.3)(vite@6.4.2(@types/node@22.19.15)(tsx@4.21.0)(yaml@2.8.3)) vitest: specifier: ^2.0.0 version: 2.1.9(@types/node@22.19.15)(jsdom@26.1.0) @@ -314,6 +314,40 @@ importers: specifier: ^2.0.0 version: 2.1.9(@types/node@22.19.15)(jsdom@26.1.0) + packages/storage-postgres: + dependencies: + "@agentrail/app": + specifier: workspace:* + version: link:../app + "@agentrail/capabilities": + specifier: workspace:* + version: link:../capabilities + "@agentrail/core": + specifier: workspace:* + version: link:../core + postgres: + specifier: ^3.4.5 + version: 3.4.9 + devDependencies: + "@testcontainers/postgresql": + specifier: ^11.14.0 + version: 11.14.0 + "@types/node": + specifier: ^22.0.0 + version: 22.19.15 + testcontainers: + specifier: ^11.14.0 + version: 11.14.0 + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 + vite-tsconfig-paths: + specifier: ^6.1.1 + version: 6.1.1(typescript@5.9.3)(vite@5.4.21(@types/node@22.19.15)) + vitest: + specifier: ^2.0.0 + version: 2.1.9(@types/node@22.19.15)(jsdom@26.1.0) + packages/testing: dependencies: "@agentrail/core": @@ -1451,6 +1485,13 @@ packages: "@types/node": optional: true + "@isaacs/cliui@8.0.2": + resolution: + { + integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==, + } + engines: { node: ">=12" } + "@jridgewell/gen-mapping@0.3.13": resolution: { @@ -1488,6 +1529,12 @@ packages: integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==, } + "@kwsites/file-exists@1.1.1": + resolution: + { + integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==, + } + "@manypkg/find-root@1.1.0": resolution: { @@ -1540,6 +1587,13 @@ packages: } engines: { node: ">= 8" } + "@pkgjs/parseargs@0.11.0": + resolution: + { + integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==, + } + engines: { node: ">=14" } + "@protobufjs/aspromise@1.1.2": resolution: { @@ -1818,6 +1872,12 @@ packages: integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==, } + "@testcontainers/postgresql@11.14.0": + resolution: + { + integrity: sha512-wYbJn8GRTj8qfqzfVubxioYWlHJU/ImIjuzPwyy9C5Qfo6g3GLduPZAj+BifvqTZjgT3gd4gFVLCPhBji7dc1w==, + } + "@types/babel__core@7.20.5": resolution: { @@ -2058,6 +2118,12 @@ packages: integrity: sha512-ShM1mz7rCjdssXt7Xz0u1/R2BJC7piWa3SJpUBiVjCf2A3XNn4cP6pUVaD8bLanpPVVn4IKzJuw3dOvkJ8IbYw==, } + "@types/dockerode@4.0.1": + resolution: + { + integrity: sha512-cmUpB+dPN955PxBEuXE3f6lKO1hHiIGYJA46IVF3BJpNsZGvtBDcRnlrHYHtOH/B6vtDOyl2kZ2ShAu3mgc27Q==, + } + "@types/dompurify@3.2.0": resolution: { @@ -2145,6 +2211,18 @@ packages: integrity: sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==, } + "@types/ssh2-streams@0.1.13": + resolution: + { + integrity: sha512-faHyY3brO9oLEA0QlcO8N2wT7R0+1sHWZvQ+y3rMLwdY1ZyS1z0W3t65j9PqT4HmQ6ALzNe7RZlNuCNE0wBSWA==, + } + + "@types/ssh2@0.5.52": + resolution: + { + integrity: sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==, + } + "@types/ssh2@1.15.5": resolution: { @@ -2372,6 +2450,13 @@ packages: } engines: { node: ">=8" } + ansi-regex@6.2.2: + resolution: + { + integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==, + } + engines: { node: ">=12" } + ansi-styles@4.3.0: resolution: { @@ -2379,6 +2464,13 @@ packages: } engines: { node: ">=8" } + ansi-styles@6.2.3: + resolution: + { + integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==, + } + engines: { node: ">=12" } + anymatch@3.1.3: resolution: { @@ -2386,6 +2478,20 @@ packages: } engines: { node: ">= 8" } + archiver-utils@5.0.2: + resolution: + { + integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==, + } + engines: { node: ">= 14" } + + archiver@7.0.1: + resolution: + { + integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==, + } + engines: { node: ">= 14" } + argparse@1.0.10: resolution: { @@ -2424,6 +2530,18 @@ packages: } engines: { node: ">=12" } + async-lock@1.4.1: + resolution: + { + integrity: sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==, + } + + async@3.2.6: + resolution: + { + integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==, + } + asynckit@0.4.0: resolution: { @@ -2447,6 +2565,12 @@ packages: integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==, } + balanced-match@1.0.2: + resolution: + { + integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==, + } + bare-events@2.8.2: resolution: { @@ -2559,6 +2683,12 @@ packages: } engines: { node: ">= 0.8", npm: 1.2.8000 || >= 1.4.16 } + brace-expansion@2.1.0: + resolution: + { + integrity: sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==, + } + braces@3.0.3: resolution: { @@ -2574,12 +2704,25 @@ packages: engines: { node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7 } hasBin: true + buffer-crc32@1.0.0: + resolution: + { + integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==, + } + engines: { node: ">=8.0.0" } + buffer@5.7.1: resolution: { integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==, } + buffer@6.0.3: + resolution: + { + integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==, + } + buildcheck@0.0.7: resolution: { @@ -2587,6 +2730,13 @@ packages: } engines: { node: ">=10.0.0" } + byline@5.0.0: + resolution: + { + integrity: sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==, + } + engines: { node: ">=0.10.0" } + bytes@3.1.2: resolution: { @@ -2780,6 +2930,13 @@ packages: } engines: { node: ^12.20.0 || >=14 } + compress-commons@6.0.2: + resolution: + { + integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==, + } + engines: { node: ">= 14" } + concurrently@9.2.1: resolution: { @@ -2860,6 +3017,13 @@ packages: engines: { node: ">=0.8" } hasBin: true + crc32-stream@6.0.0: + resolution: + { + integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==, + } + engines: { node: ">= 14" } + cross-spawn@7.0.6: resolution: { @@ -3274,6 +3438,13 @@ packages: } engines: { node: ">=8" } + docker-compose@1.4.2: + resolution: + { + integrity: sha512-rPHigTKGaEHpkUmfd69QgaOp+Os5vGJwG/Ry8lcr8W/382AmI+z/D7qoa9BybKIkqNppaIbs8RYeHSevdQjWww==, + } + engines: { node: ">= 6.0.0" } + docker-modem@5.0.7: resolution: { @@ -3314,6 +3485,12 @@ packages: } engines: { node: ">= 0.4" } + eastasianwidth@0.2.0: + resolution: + { + integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==, + } + ee-first@1.1.1: resolution: { @@ -3332,6 +3509,12 @@ packages: integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==, } + emoji-regex@9.2.2: + resolution: + { + integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==, + } + encodeurl@2.0.0: resolution: { @@ -3483,6 +3666,13 @@ packages: integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==, } + events@3.3.0: + resolution: + { + integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==, + } + engines: { node: ">=0.8.x" } + expect-type@1.3.0: resolution: { @@ -3561,6 +3751,13 @@ packages: } engines: { node: ">=8" } + foreground-child@3.3.1: + resolution: + { + integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==, + } + engines: { node: ">=14" } + form-data-encoder@1.7.2: resolution: { @@ -3665,6 +3862,13 @@ packages: } engines: { node: ">= 0.4" } + get-port@7.2.0: + resolution: + { + integrity: sha512-afP4W205ONCuMoPBqcR6PSXnzX35KTcJygfJfcp+QY+uwm3p20p1YczWXhlICIzGMCxYBQcySEcOgsJcrkyobg==, + } + engines: { node: ">=16" } + get-proto@1.0.1: resolution: { @@ -3685,6 +3889,14 @@ packages: } engines: { node: ">= 6" } + glob@10.5.0: + resolution: + { + integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==, + } + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + hasBin: true + globby@11.1.0: resolution: { @@ -3955,6 +4167,13 @@ packages: integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==, } + is-stream@2.0.1: + resolution: + { + integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==, + } + engines: { node: ">=8" } + is-subdir@1.2.0: resolution: { @@ -3981,6 +4200,12 @@ packages: integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==, } + jackspeak@3.4.3: + resolution: + { + integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==, + } + js-tokens@4.0.0: resolution: { @@ -4073,6 +4298,13 @@ packages: integrity: sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==, } + lazystream@1.0.1: + resolution: + { + integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==, + } + engines: { node: ">= 0.6.3" } + lie@3.3.0: resolution: { @@ -4104,6 +4336,12 @@ packages: integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==, } + lodash@4.18.1: + resolution: + { + integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==, + } + long@5.3.2: resolution: { @@ -4495,12 +4733,41 @@ packages: engines: { node: ">=4" } hasBin: true + minimatch@5.1.9: + resolution: + { + integrity: sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==, + } + engines: { node: ">=10" } + + minimatch@9.0.9: + resolution: + { + integrity: sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==, + } + engines: { node: ">=16 || 14 >=14.17" } + + minipass@7.1.3: + resolution: + { + integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==, + } + engines: { node: ">=16 || 14 >=14.17" } + mkdirp-classic@0.5.3: resolution: { integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==, } + mkdirp@3.0.1: + resolution: + { + integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==, + } + engines: { node: ">=10" } + hasBin: true + mlly@1.8.2: resolution: { @@ -4681,6 +4948,12 @@ packages: } engines: { node: ">=6" } + package-json-from-dist@1.0.1: + resolution: + { + integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==, + } + package-manager-detector@0.2.11: resolution: { @@ -4745,6 +5018,13 @@ packages: } engines: { node: ">=8" } + path-scurry@1.11.1: + resolution: + { + integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==, + } + engines: { node: ">=16 || 14 >=14.18" } + path-to-regexp@0.1.13: resolution: { @@ -4852,6 +5132,13 @@ packages: } engines: { node: ^10 || ^12 || >=14 } + postgres@3.4.9: + resolution: + { + integrity: sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw==, + } + engines: { node: ">=12" } + prettier-plugin-organize-imports@4.3.0: resolution: { @@ -4887,6 +5174,26 @@ packages: integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==, } + process@0.11.10: + resolution: + { + integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==, + } + engines: { node: ">= 0.6.0" } + + proper-lockfile@4.1.2: + resolution: + { + integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==, + } + + properties-reader@3.0.1: + resolution: + { + integrity: sha512-WPn+h9RGEExOKdu4bsF4HksG/uzd3cFq3MFtq8PsFeExPse5Ha/VOjQNyHhjboBFwGXGev6muJYTSPAOkROq2g==, + } + engines: { node: ">=18" } + property-information@7.1.0: resolution: { @@ -5011,6 +5318,19 @@ packages: } engines: { node: ">= 6" } + readable-stream@4.7.0: + resolution: + { + integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==, + } + engines: { node: ^12.22.0 || ^14.17.0 || >=16.0.0 } + + readdir-glob@1.1.3: + resolution: + { + integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==, + } + readdirp@3.6.0: resolution: { @@ -5062,6 +5382,13 @@ packages: integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==, } + retry@0.12.0: + resolution: + { + integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==, + } + engines: { node: ">= 4" } + reusify@1.1.0: resolution: { @@ -5240,6 +5567,12 @@ packages: integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==, } + signal-exit@3.0.7: + resolution: + { + integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==, + } + signal-exit@4.1.0: resolution: { @@ -5298,6 +5631,12 @@ packages: } engines: { node: ">=0.8" } + ssh-remote-port-forward@1.0.4: + resolution: + { + integrity: sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==, + } + ssh2@1.17.0: resolution: { @@ -5343,6 +5682,13 @@ packages: } engines: { node: ">=8" } + string-width@5.1.2: + resolution: + { + integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==, + } + engines: { node: ">=12" } + string_decoder@1.1.1: resolution: { @@ -5368,6 +5714,13 @@ packages: } engines: { node: ">=8" } + strip-ansi@7.2.0: + resolution: + { + integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==, + } + engines: { node: ">=12" } + strip-bom@3.0.0: resolution: { @@ -5419,6 +5772,12 @@ packages: integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==, } + tar-fs@3.1.2: + resolution: + { + integrity: sha512-QGxxTxxyleAdyM3kpFs14ymbYmNFrfY+pHj7Z8FgtbZ7w2//VAgLMac7sT6nRpIHjppXO2AwwEOg0bPFVRcmXw==, + } + tar-stream@2.2.0: resolution: { @@ -5445,6 +5804,12 @@ packages: } engines: { node: ">=8" } + testcontainers@11.14.0: + resolution: + { + integrity: sha512-r9pniwv/iwzyHaI7gwAvAm4Y+IvjJg3vBWdjrUCaDMc2AXIr4jKbq7jJO18Mw2ybs73pZy1Aj7p/4RVBGMRWjg==, + } + text-decoder@1.2.7: resolution: { @@ -5518,6 +5883,13 @@ packages: } hasBin: true + tmp@0.2.5: + resolution: + { + integrity: sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==, + } + engines: { node: ">=14.14" } + to-regex-range@5.0.1: resolution: { @@ -5665,6 +6037,13 @@ packages: integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==, } + undici@7.24.8: + resolution: + { + integrity: sha512-6KQ/+QxK49Z/p3HO6E5ZCZWNnCasyZLa5ExaVYyvPxUwKtbCPMKELJOqh7EqOle0t9cH/7d2TaaTRRa6Nhs4YQ==, + } + engines: { node: ">=20.18.1" } + unified@11.0.5: resolution: { @@ -6065,6 +6444,13 @@ packages: } engines: { node: ">=10" } + wrap-ansi@8.1.0: + resolution: + { + integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==, + } + engines: { node: ">=12" } + wrappy@1.0.2: resolution: { @@ -6149,6 +6535,13 @@ packages: } engines: { node: ">=12" } + zip-stream@6.0.1: + resolution: + { + integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==, + } + engines: { node: ">= 14" } + zwitch@2.0.4: resolution: { @@ -6768,6 +7161,15 @@ snapshots: optionalDependencies: "@types/node": 22.19.15 + "@isaacs/cliui@8.0.2": + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.2.0 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + "@jridgewell/gen-mapping@0.3.13": dependencies: "@jridgewell/sourcemap-codec": 1.5.5 @@ -6789,6 +7191,12 @@ snapshots: "@js-sdsl/ordered-map@4.4.2": {} + "@kwsites/file-exists@1.1.1": + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + "@manypkg/find-root@1.1.0": dependencies: "@babel/runtime": 7.29.2 @@ -6825,6 +7233,9 @@ snapshots: "@nodelib/fs.scandir": 2.1.5 fastq: 1.20.1 + "@pkgjs/parseargs@0.11.0": + optional: true + "@protobufjs/aspromise@1.1.2": {} "@protobufjs/base64@1.1.2": {} @@ -6929,6 +7340,15 @@ snapshots: "@standard-schema/spec@1.1.0": {} + "@testcontainers/postgresql@11.14.0": + dependencies: + testcontainers: 11.14.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + - supports-color + "@types/babel__core@7.20.5": dependencies: "@babel/parser": 7.29.2 @@ -7089,6 +7509,12 @@ snapshots: "@types/node": 22.19.15 "@types/ssh2": 1.15.5 + "@types/dockerode@4.0.1": + dependencies: + "@types/docker-modem": 3.0.6 + "@types/node": 22.19.15 + "@types/ssh2": 1.15.5 + "@types/dompurify@3.2.0": dependencies: dompurify: 3.3.3 @@ -7140,6 +7566,15 @@ snapshots: dependencies: csstype: 3.2.3 + "@types/ssh2-streams@0.1.13": + dependencies: + "@types/node": 22.19.15 + + "@types/ssh2@0.5.52": + dependencies: + "@types/node": 22.19.15 + "@types/ssh2-streams": 0.1.13 + "@types/ssh2@1.15.5": dependencies: "@types/node": 18.19.130 @@ -7284,15 +7719,43 @@ snapshots: ansi-regex@5.0.1: {} + ansi-regex@6.2.2: {} + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 + ansi-styles@6.2.3: {} + anymatch@3.1.3: dependencies: normalize-path: 3.0.0 picomatch: 2.3.2 + archiver-utils@5.0.2: + dependencies: + glob: 10.5.0 + graceful-fs: 4.2.11 + is-stream: 2.0.1 + lazystream: 1.0.1 + lodash: 4.18.1 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + + archiver@7.0.1: + dependencies: + archiver-utils: 5.0.2 + async: 3.2.6 + buffer-crc32: 1.0.0 + readable-stream: 4.7.0 + readdir-glob: 1.1.3 + tar-stream: 3.1.8 + zip-stream: 6.0.1 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + argparse@1.0.10: dependencies: sprintf-js: 1.0.3 @@ -7309,12 +7772,18 @@ snapshots: assertion-error@2.0.1: {} + async-lock@1.4.1: {} + + async@3.2.6: {} + asynckit@0.4.0: {} b4a@1.8.0: {} bail@2.0.2: {} + balanced-match@1.0.2: {} + bare-events@2.8.2: {} bare-fs@4.5.6: @@ -7386,6 +7855,10 @@ snapshots: transitivePeerDependencies: - supports-color + brace-expansion@2.1.0: + dependencies: + balanced-match: 1.0.2 + braces@3.0.3: dependencies: fill-range: 7.1.1 @@ -7398,14 +7871,23 @@ snapshots: node-releases: 2.0.36 update-browserslist-db: 1.2.3(browserslist@4.28.1) + buffer-crc32@1.0.0: {} + buffer@5.7.1: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + buildcheck@0.0.7: optional: true + byline@5.0.0: {} + bytes@3.1.2: {} cac@6.7.14: {} @@ -7510,6 +7992,14 @@ snapshots: commander@9.5.0: {} + compress-commons@6.0.2: + dependencies: + crc-32: 1.2.2 + crc32-stream: 6.0.0 + is-stream: 2.0.1 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + concurrently@9.2.1: dependencies: chalk: 4.1.2 @@ -7551,6 +8041,11 @@ snapshots: crc-32@1.2.2: {} + crc32-stream@6.0.0: + dependencies: + crc-32: 1.2.2 + readable-stream: 4.7.0 + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -7797,6 +8292,10 @@ snapshots: dependencies: path-type: 4.0.0 + docker-compose@1.4.2: + dependencies: + yaml: 2.8.3 + docker-modem@5.0.7: dependencies: debug: 4.4.3 @@ -7834,12 +8333,16 @@ snapshots: es-errors: 1.3.0 gopd: 1.2.0 + eastasianwidth@0.2.0: {} + ee-first@1.1.1: {} electron-to-chromium@1.5.328: {} emoji-regex@8.0.0: {} + emoji-regex@9.2.2: {} + encodeurl@2.0.0: {} end-of-stream@1.4.5: @@ -7980,6 +8483,8 @@ snapshots: transitivePeerDependencies: - bare-abort-controller + events@3.3.0: {} + expect-type@1.3.0: {} express@4.22.1: @@ -8061,6 +8566,11 @@ snapshots: locate-path: 5.0.0 path-exists: 4.0.0 + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + form-data-encoder@1.7.2: {} form-data@4.0.5: @@ -8121,6 +8631,8 @@ snapshots: hasown: 2.0.2 math-intrinsics: 1.1.0 + get-port@7.2.0: {} + get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 @@ -8134,6 +8646,15 @@ snapshots: dependencies: is-glob: 4.0.3 + glob@10.5.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.9 + minipass: 7.1.3 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + globby@11.1.0: dependencies: array-union: 2.1.0 @@ -8280,6 +8801,8 @@ snapshots: is-potential-custom-element-name@1.0.1: {} + is-stream@2.0.1: {} + is-subdir@1.2.0: dependencies: better-path-resolve: 1.0.0 @@ -8290,6 +8813,12 @@ snapshots: isexe@2.0.0: {} + jackspeak@3.4.3: + dependencies: + "@isaacs/cliui": 8.0.2 + optionalDependencies: + "@pkgjs/parseargs": 0.11.0 + js-tokens@4.0.0: {} js-yaml@3.14.2: @@ -8361,6 +8890,10 @@ snapshots: layout-base@2.0.1: {} + lazystream@1.0.1: + dependencies: + readable-stream: 2.3.8 + lie@3.3.0: dependencies: immediate: 3.0.6 @@ -8375,6 +8908,8 @@ snapshots: lodash.startcase@4.4.0: {} + lodash@4.18.1: {} + long@5.3.2: {} longest-streak@3.1.0: {} @@ -8805,8 +9340,20 @@ snapshots: mime@1.6.0: {} + minimatch@5.1.9: + dependencies: + brace-expansion: 2.1.0 + + minimatch@9.0.9: + dependencies: + brace-expansion: 2.1.0 + + minipass@7.1.3: {} + mkdirp-classic@0.5.3: {} + mkdirp@3.0.1: {} + mlly@1.8.2: dependencies: acorn: 8.16.0 @@ -8887,6 +9434,8 @@ snapshots: p-try@2.2.0: {} + package-json-from-dist@1.0.1: {} + package-manager-detector@0.2.11: dependencies: quansync: 0.2.11 @@ -8919,6 +9468,11 @@ snapshots: path-key@3.1.1: {} + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.3 + path-to-regexp@0.1.13: {} path-type@4.0.0: {} @@ -8968,6 +9522,8 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postgres@3.4.9: {} + prettier-plugin-organize-imports@4.3.0(prettier@3.8.1)(typescript@5.9.3): dependencies: prettier: 3.8.1 @@ -8979,6 +9535,21 @@ snapshots: process-nextick-args@2.0.1: {} + process@0.11.10: {} + + proper-lockfile@4.1.2: + dependencies: + graceful-fs: 4.2.11 + retry: 0.12.0 + signal-exit: 3.0.7 + + properties-reader@3.0.1: + dependencies: + "@kwsites/file-exists": 1.1.1 + mkdirp: 3.0.1 + transitivePeerDependencies: + - supports-color + property-information@7.1.0: {} protobufjs@7.5.4: @@ -9077,6 +9648,18 @@ snapshots: string_decoder: 1.3.0 util-deprecate: 1.0.2 + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + readdir-glob@1.1.3: + dependencies: + minimatch: 5.1.9 + readdirp@3.6.0: dependencies: picomatch: 2.3.2 @@ -9121,6 +9704,8 @@ snapshots: resolve-pkg-maps@1.0.0: {} + retry@0.12.0: {} + reusify@1.1.0: {} robust-predicates@3.0.3: {} @@ -9260,6 +9845,8 @@ snapshots: siginfo@2.0.0: {} + signal-exit@3.0.7: {} + signal-exit@4.1.0: {} sisteransi@1.0.5: {} @@ -9283,6 +9870,11 @@ snapshots: dependencies: frac: 1.1.2 + ssh-remote-port-forward@1.0.4: + dependencies: + "@types/ssh2": 0.5.52 + ssh2: 1.17.0 + ssh2@1.17.0: dependencies: asn1: 0.2.6 @@ -9314,6 +9906,12 @@ snapshots: is-fullwidth-code-point: 3.0.0 strip-ansi: 6.0.1 + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.2.0 + string_decoder@1.1.1: dependencies: safe-buffer: 5.1.2 @@ -9331,6 +9929,10 @@ snapshots: dependencies: ansi-regex: 5.0.1 + strip-ansi@7.2.0: + dependencies: + ansi-regex: 6.2.2 + strip-bom@3.0.0: {} style-to-js@1.1.21: @@ -9360,6 +9962,18 @@ snapshots: pump: 3.0.4 tar-stream: 2.2.0 + tar-fs@3.1.2: + dependencies: + pump: 3.0.4 + tar-stream: 3.1.8 + optionalDependencies: + bare-fs: 4.5.6 + bare-path: 3.0.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + tar-stream@2.2.0: dependencies: bl: 4.1.0 @@ -9388,6 +10002,29 @@ snapshots: term-size@2.2.1: {} + testcontainers@11.14.0: + dependencies: + "@balena/dockerignore": 1.0.2 + "@types/dockerode": 4.0.1 + archiver: 7.0.1 + async-lock: 1.4.1 + byline: 5.0.0 + debug: 4.4.3 + docker-compose: 1.4.2 + dockerode: 4.0.10 + get-port: 7.2.0 + proper-lockfile: 4.1.2 + properties-reader: 3.0.1 + ssh-remote-port-forward: 1.0.4 + tar-fs: 3.1.2 + tmp: 0.2.5 + undici: 7.24.8 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + - supports-color + text-decoder@1.2.7: dependencies: b4a: 1.8.0 @@ -9419,6 +10056,8 @@ snapshots: dependencies: tldts-core: 6.1.86 + tmp@0.2.5: {} + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 @@ -9487,6 +10126,8 @@ snapshots: undici-types@6.21.0: {} + undici@7.24.8: {} + unified@11.0.5: dependencies: "@types/unist": 3.0.3 @@ -9737,6 +10378,12 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.3 + string-width: 5.1.2 + strip-ansi: 7.2.0 + wrappy@1.0.2: {} ws@8.20.0: {} @@ -9775,4 +10422,10 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + zip-stream@6.0.1: + dependencies: + archiver-utils: 5.0.2 + compress-commons: 6.0.2 + readable-stream: 4.7.0 + zwitch@2.0.4: {} From 2ef31f92d90659c3627ad03f520e198670c3a782 Mon Sep 17 00:00:00 2001 From: yai-dev Date: Mon, 13 Apr 2026 18:13:56 +0800 Subject: [PATCH 2/2] docs: add build-a-storage-backend to sidebar nav Signed-off-by: yai-dev --- docs/.vitepress/config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index f658834..e3e612c 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -99,6 +99,7 @@ export default defineConfig({ { text: "Tool Permissions", link: "/guides/tool-permissions" }, { text: "Add Context", link: "/guides/add-context" }, { text: "Configure Sessions", link: "/guides/configure-sessions" }, + { text: "Build a Storage Backend", link: "/guides/build-a-storage-backend" }, { text: "Consume Stream (SSE)", link: "/guides/consume-stream" }, { text: "Write a Plugin", link: "/guides/write-a-plugin" }, {