From bec3bf6423df895982a4583e629a6b024fa3e4f1 Mon Sep 17 00:00:00 2001 From: bashybaranaba Date: Thu, 20 Aug 2026 14:14:19 +0300 Subject: [PATCH 01/35] feat: modularize skills artifacts and providers --- .changeset/modular-skills-artifacts.md | 5 + apps/commons-api/.env.example | 10 + apps/commons-api/Dockerfile | 1 + .../versioned/019_agent_skill_assignments.sql | 36 + .../versioned/020_capability_providers.sql | 28 + .../migrations/versioned/021_ui_plugins.sql | 21 + apps/commons-api/models/schema.ts | 129 +++ .../build-commons-ui-plugin/SKILL.md | 27 + apps/commons-api/src/agent/agent.service.ts | 107 ++- apps/commons-api/src/app.module.ts | 4 + .../src/computer/computer.module.ts | 3 +- .../src/computer/computer.service.spec.ts | 5 +- .../src/computer/computer.service.ts | 122 ++- .../src/files/files.service.spec.ts | 83 +- apps/commons-api/src/files/files.service.ts | 436 +++++++-- .../src/files/library.controller.ts | 2 + apps/commons-api/src/files/library.service.ts | 34 +- .../capability-provider.controller.ts | 51 + .../provider/capability-provider.module.ts | 10 + .../capability-provider.service.spec.ts | 38 + .../provider/capability-provider.service.ts | 292 ++++++ apps/commons-api/src/provider/index.ts | 2 + .../commons-api/src/skill/skill.controller.ts | 95 +- .../src/skill/skill.service.spec.ts | 64 ++ apps/commons-api/src/skill/skill.service.ts | 553 ++++++++++- apps/commons-api/src/tool/tool.module.ts | 4 + .../src/tool/tools/common-tool.service.ts | 83 +- .../tool/tools/web-search.provider.spec.ts | 85 ++ .../src/tool/tools/web-search.provider.ts | 163 +++- apps/commons-api/src/ui-plugin/index.ts | 2 + .../src/ui-plugin/ui-plugin.controller.ts | 73 ++ .../src/ui-plugin/ui-plugin.module.ts | 10 + .../src/ui-plugin/ui-plugin.service.spec.ts | 68 ++ .../src/ui-plugin/ui-plugin.service.ts | 254 +++++ apps/commons-api/src/wallet/dto/wallet.dto.ts | 6 +- apps/commons-api/src/wallet/wallet.module.ts | 3 +- apps/commons-api/src/wallet/wallet.service.ts | 184 +++- .../app/api/providers/[capability]/route.ts | 50 + apps/commons-app/app/api/providers/route.ts | 29 + .../[skillId]/agents/[agentId]/route.ts | 12 + .../app/api/skills/agents/[agentId]/route.ts | 9 + .../app/api/skills/import/route.ts | 31 + .../app/api/ui-plugins/[pluginId]/route.ts | 29 + .../api/ui-plugins/[pluginId]/status/route.ts | 36 + apps/commons-app/app/api/ui-plugins/route.ts | 41 + .../app/api/ui-plugins/slug/[slug]/route.ts | 23 + apps/commons-app/app/apps/[slug]/page.tsx | 79 ++ apps/commons-app/app/layout.tsx | 2 + apps/commons-app/app/library/page.tsx | 18 + apps/commons-app/app/studio/[tab]/page.tsx | 46 +- .../app/studio/agents/[agent]/page.tsx | 376 +++----- apps/commons-app/app/studio/apps/page.tsx | 1 + .../components/account/settings-panel.tsx | 375 +++++++- .../artifacts/agent-artifacts-view.tsx | 240 +++++ .../copilot/floating-commons-copilot.tsx | 61 +- .../components/layout/dashboard-bar.tsx | 33 +- .../components/layout/dashboard-side-bar.tsx | 18 +- .../components/plugins/plugin-frame.tsx | 70 ++ .../components/plugins/plugin-widget-host.tsx | 89 ++ apps/commons-app/components/plugins/types.ts | 20 + .../components/plugins/ui-plugins-view.tsx | 160 ++++ .../skills/skills-marketplace-view.tsx | 882 +++++++++++++++--- apps/commons-app/hooks/use-skills.ts | 94 +- apps/commons-app/package.json | 2 +- buildspec.aws.yml | 2 +- docs/architecture/modular-capabilities.md | 83 ++ infra/aws/README.md | 10 +- packages/commons-sdk/dist/index.cjs | 64 +- packages/commons-sdk/dist/index.cjs.map | 2 +- packages/commons-sdk/dist/index.d.mts | 124 ++- packages/commons-sdk/dist/index.d.ts | 124 ++- packages/commons-sdk/dist/index.mjs | 64 +- packages/commons-sdk/dist/index.mjs.map | 2 +- packages/commons-sdk/src/client.test.ts | 57 ++ packages/commons-sdk/src/client.ts | 112 ++- packages/commons-sdk/src/index.ts | 10 + packages/commons-sdk/src/types.ts | 94 ++ pnpm-lock.yaml | 4 +- 78 files changed, 6003 insertions(+), 668 deletions(-) create mode 100644 .changeset/modular-skills-artifacts.md create mode 100644 apps/commons-api/migrations/versioned/019_agent_skill_assignments.sql create mode 100644 apps/commons-api/migrations/versioned/020_capability_providers.sql create mode 100644 apps/commons-api/migrations/versioned/021_ui_plugins.sql create mode 100644 apps/commons-api/platform-skills/build-commons-ui-plugin/SKILL.md create mode 100644 apps/commons-api/src/provider/capability-provider.controller.ts create mode 100644 apps/commons-api/src/provider/capability-provider.module.ts create mode 100644 apps/commons-api/src/provider/capability-provider.service.spec.ts create mode 100644 apps/commons-api/src/provider/capability-provider.service.ts create mode 100644 apps/commons-api/src/provider/index.ts create mode 100644 apps/commons-api/src/ui-plugin/index.ts create mode 100644 apps/commons-api/src/ui-plugin/ui-plugin.controller.ts create mode 100644 apps/commons-api/src/ui-plugin/ui-plugin.module.ts create mode 100644 apps/commons-api/src/ui-plugin/ui-plugin.service.spec.ts create mode 100644 apps/commons-api/src/ui-plugin/ui-plugin.service.ts create mode 100644 apps/commons-app/app/api/providers/[capability]/route.ts create mode 100644 apps/commons-app/app/api/providers/route.ts create mode 100644 apps/commons-app/app/api/skills/[skillId]/agents/[agentId]/route.ts create mode 100644 apps/commons-app/app/api/skills/agents/[agentId]/route.ts create mode 100644 apps/commons-app/app/api/skills/import/route.ts create mode 100644 apps/commons-app/app/api/ui-plugins/[pluginId]/route.ts create mode 100644 apps/commons-app/app/api/ui-plugins/[pluginId]/status/route.ts create mode 100644 apps/commons-app/app/api/ui-plugins/route.ts create mode 100644 apps/commons-app/app/api/ui-plugins/slug/[slug]/route.ts create mode 100644 apps/commons-app/app/apps/[slug]/page.tsx create mode 100644 apps/commons-app/app/studio/apps/page.tsx create mode 100644 apps/commons-app/components/artifacts/agent-artifacts-view.tsx create mode 100644 apps/commons-app/components/plugins/plugin-frame.tsx create mode 100644 apps/commons-app/components/plugins/plugin-widget-host.tsx create mode 100644 apps/commons-app/components/plugins/types.ts create mode 100644 apps/commons-app/components/plugins/ui-plugins-view.tsx create mode 100644 docs/architecture/modular-capabilities.md diff --git a/.changeset/modular-skills-artifacts.md b/.changeset/modular-skills-artifacts.md new file mode 100644 index 00000000..f7d9bea1 --- /dev/null +++ b/.changeset/modular-skills-artifacts.md @@ -0,0 +1,5 @@ +--- +"@agent-commons/sdk": minor +--- + +Add per-agent skill assignments, agent-scoped library items, configurable capability providers, portable skill imports, and sandboxed UI plugin management. diff --git a/apps/commons-api/.env.example b/apps/commons-api/.env.example index 53415afb..c8aea2dc 100644 --- a/apps/commons-api/.env.example +++ b/apps/commons-api/.env.example @@ -102,6 +102,16 @@ BRAVE_SEARCH_COST_USD_PER_CALL="0.005" # SEARXNG_API_KEY="" # SEARXNG_SEARCH_COST_USD_PER_CALL="0" OPENAI_TRANSCRIPTION_COST_USD_PER_MINUTE="0.003" +# Video uploads are sampled with ffmpeg and summarized by a multimodal model. +# Set to false to retain videos without automatic understanding. +AGENT_FILE_VIDEO_UNDERSTANDING_ENABLED="true" +AGENT_FILE_VIDEO_UNDERSTANDING_MODEL="gpt-5.4-mini" +AGENT_FILE_VIDEO_MAX_FRAMES="8" +FFMPEG_PATH="ffmpeg" +# Audio-only transcription remains opt-in. Video audio is transcribed as part +# of enabled video understanding. +AGENT_FILE_AUDIO_TRANSCRIPTION_ENABLED="false" +AGENT_FILE_AUDIO_TRANSCRIPTION_MODEL="gpt-4o-mini-transcribe" OPENAI_TTS_COST_USD_PER_1K_CHARACTERS="0.015" ELEVENLABS_TTS_COST_USD_PER_1K_CHARACTERS="0.10" # Optional JSON override keyed by quality then size; defaults track GPT Image 2. diff --git a/apps/commons-api/Dockerfile b/apps/commons-api/Dockerfile index 141b0d54..dc003783 100644 --- a/apps/commons-api/Dockerfile +++ b/apps/commons-api/Dockerfile @@ -27,6 +27,7 @@ FROM public.ecr.aws/docker/library/node:22.11.0-bookworm-slim AS runtime RUN apt-get update && apt-get install -y \ chromium \ chromium-sandbox \ + ffmpeg \ fonts-liberation \ fonts-noto-color-emoji \ libatk-bridge2.0-0 \ diff --git a/apps/commons-api/migrations/versioned/019_agent_skill_assignments.sql b/apps/commons-api/migrations/versioned/019_agent_skill_assignments.sql new file mode 100644 index 00000000..d69546e8 --- /dev/null +++ b/apps/commons-api/migrations/versioned/019_agent_skill_assignments.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS agent_skill ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + agent_id text NOT NULL REFERENCES agent(agent_id) ON DELETE CASCADE, + skill_id text NOT NULL REFERENCES skill(skill_id) ON DELETE CASCADE, + is_enabled boolean NOT NULL DEFAULT true, + assigned_by text, + config jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT timezone('utc', now()), + updated_at timestamptz NOT NULL DEFAULT timezone('utc', now()) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_skill_agent_skill + ON agent_skill (agent_id, skill_id); + +CREATE INDEX IF NOT EXISTS idx_agent_skill_skill + ON agent_skill (skill_id, agent_id); + +-- Preserve existing agent-owned skills as explicit assignments. +INSERT INTO agent_skill (agent_id, skill_id, assigned_by) +SELECT a.agent_id, s.skill_id, COALESCE(a.owner_user_id, a.owner) +FROM skill s +INNER JOIN agent a ON a.agent_id = s.owner_id +WHERE s.owner_type = 'agent' +ON CONFLICT (agent_id, skill_id) DO NOTHING; + +-- Bundled Commons skills start on each account's Commons Copilot. Other +-- agents receive them only when the user explicitly enables them. +INSERT INTO agent_skill (agent_id, skill_id, assigned_by) +SELECT a.agent_id, s.skill_id, COALESCE(a.owner_user_id, a.owner) +FROM agent a +CROSS JOIN skill s +WHERE a.is_default = true + AND a.is_system_managed = true + AND s.owner_type = 'platform' + AND s.is_active = true +ON CONFLICT (agent_id, skill_id) DO NOTHING; diff --git a/apps/commons-api/migrations/versioned/020_capability_providers.sql b/apps/commons-api/migrations/versioned/020_capability_providers.sql new file mode 100644 index 00000000..ead62154 --- /dev/null +++ b/apps/commons-api/migrations/versioned/020_capability_providers.sql @@ -0,0 +1,28 @@ +CREATE TABLE IF NOT EXISTS capability_provider ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + owner_id text NOT NULL, + workspace_id text, + capability text NOT NULL, + provider text NOT NULL, + display_name text, + endpoint_url text, + settings jsonb NOT NULL DEFAULT '{}'::jsonb, + encrypted_credentials text, + credentials_iv text, + credentials_tag text, + status text NOT NULL DEFAULT 'active', + created_at timestamptz NOT NULL DEFAULT timezone('utc', now()), + updated_at timestamptz NOT NULL DEFAULT timezone('utc', now()), + CONSTRAINT capability_provider_status_check + CHECK (status IN ('active', 'disabled', 'error')) +); + +CREATE UNIQUE INDEX IF NOT EXISTS capability_provider_owner_capability_idx + ON capability_provider (owner_id, capability); + +CREATE INDEX IF NOT EXISTS capability_provider_provider_idx + ON capability_provider (capability, provider); + +ALTER TABLE agent_wallet + ADD COLUMN IF NOT EXISTS provider text NOT NULL DEFAULT 'commons_mpc', + ADD COLUMN IF NOT EXISTS provider_wallet_id text; diff --git a/apps/commons-api/migrations/versioned/021_ui_plugins.sql b/apps/commons-api/migrations/versioned/021_ui_plugins.sql new file mode 100644 index 00000000..b1a3b4f6 --- /dev/null +++ b/apps/commons-api/migrations/versioned/021_ui_plugins.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS "ui_plugin" ( + "plugin_id" uuid PRIMARY KEY DEFAULT uuid_generate_v4() NOT NULL, + "owner_user_id" text NOT NULL, + "workspace_id" text, + "created_by_agent_id" text REFERENCES "agent"("agent_id") ON DELETE SET NULL, + "code_project_id" uuid NOT NULL REFERENCES "code_project"("project_id") ON DELETE CASCADE, + "name" text NOT NULL, + "slug" text NOT NULL, + "description" text, + "version" text DEFAULT '1.0.0' NOT NULL, + "entry_url" text NOT NULL, + "manifest" jsonb NOT NULL, + "status" text DEFAULT 'draft' NOT NULL, + "created_at" timestamptz DEFAULT timezone('utc', now()) NOT NULL, + "updated_at" timestamptz DEFAULT timezone('utc', now()) NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "idx_ui_plugin_owner_slug" + ON "ui_plugin" ("owner_user_id", "slug"); +CREATE INDEX IF NOT EXISTS "idx_ui_plugin_owner_status" + ON "ui_plugin" ("owner_user_id", "status", "updated_at"); diff --git a/apps/commons-api/models/schema.ts b/apps/commons-api/models/schema.ts index e835fc66..e1cecf1f 100644 --- a/apps/commons-api/models/schema.ts +++ b/apps/commons-api/models/schema.ts @@ -158,6 +158,8 @@ export const agentWallet = pgTable('agent_wallet', { // 'erc4337' — ERC-4337 smart account with session key // 'external' — owner-connected wallet (platform holds no key) walletType: text('wallet_type').notNull().default('eoa'), + provider: text('provider').notNull().default('commons_mpc'), + providerWalletId: text('provider_wallet_id'), // The public wallet address (safe to store in plaintext) address: text('address').notNull(), @@ -300,6 +302,61 @@ export const codeProjectDeployment = pgTable( }), ); +/* ───────────────────────── UI PLUGINS ───────────────────────── */ + +export const uiPlugin = pgTable( + 'ui_plugin', + { + pluginId: uuid('plugin_id') + .default(sql`uuid_generate_v4()`) + .primaryKey(), + ownerUserId: text('owner_user_id').notNull(), + workspaceId: text('workspace_id'), + createdByAgentId: text('created_by_agent_id').references( + () => agent.agentId, + { onDelete: 'set null' }, + ), + codeProjectId: uuid('code_project_id') + .notNull() + .references(() => codeProject.projectId, { onDelete: 'cascade' }), + name: text('name').notNull(), + slug: text('slug').notNull(), + description: text('description'), + version: text('version').default('1.0.0').notNull(), + entryUrl: text('entry_url').notNull(), + manifest: jsonb('manifest') + .$type<{ + schemaVersion: '1'; + surfaces: Array<{ + type: 'page' | 'widget'; + title?: string; + width?: number; + height?: number; + }>; + permissions: Array<'theme.read' | 'navigation'>; + }>() + .notNull(), + status: text('status').default('draft').notNull(), + createdAt: timestamp('created_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + }, + (table) => ({ + ownerSlugIdx: uniqueIndex('idx_ui_plugin_owner_slug').on( + table.ownerUserId, + table.slug, + ), + ownerStatusIdx: index('idx_ui_plugin_owner_status').on( + table.ownerUserId, + table.status, + table.updatedAt, + ), + }), +); + /* ───────────────────────── AGENT COMPUTER ───────────────────────── */ export const agentComputerConfig = pgTable( @@ -2545,6 +2602,78 @@ export const skill = pgTable('skill', { .notNull(), }); +/** Explicit availability of a reusable skill on a particular agent. */ +export const agentSkill = pgTable( + 'agent_skill', + { + id: uuid('id') + .default(sql`gen_random_uuid()`) + .primaryKey(), + agentId: text('agent_id') + .notNull() + .references(() => agent.agentId, { onDelete: 'cascade' }), + skillId: text('skill_id') + .notNull() + .references(() => skill.skillId, { onDelete: 'cascade' }), + isEnabled: pgBoolean('is_enabled').default(true).notNull(), + assignedBy: text('assigned_by'), + config: jsonb('config').$type>().default({}), + createdAt: timestamp('created_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + }, + (table) => ({ + agentSkillIdx: uniqueIndex('idx_agent_skill_agent_skill').on( + table.agentId, + table.skillId, + ), + skillIdx: index('idx_agent_skill_skill').on(table.skillId, table.agentId), + }), +); + +/** + * Account-level adapters for swappable platform capabilities. Credentials are + * encrypted as one JSON envelope so provider-specific secret shapes never + * leak into the public settings document. + */ +export const capabilityProvider = pgTable( + 'capability_provider', + { + id: uuid('id') + .default(sql`gen_random_uuid()`) + .primaryKey(), + ownerId: text('owner_id').notNull(), + workspaceId: text('workspace_id'), + capability: text('capability').notNull(), + provider: text('provider').notNull(), + displayName: text('display_name'), + endpointUrl: text('endpoint_url'), + settings: jsonb('settings').$type>().default({}), + encryptedCredentials: text('encrypted_credentials'), + credentialsIv: text('credentials_iv'), + credentialsTag: text('credentials_tag'), + status: text('status').notNull().default('active'), + createdAt: timestamp('created_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }) + .default(sql`timezone('utc', now())`) + .notNull(), + }, + (table) => ({ + ownerCapabilityIdx: uniqueIndex( + 'capability_provider_owner_capability_idx', + ).on(table.ownerId, table.capability), + providerIdx: index('capability_provider_provider_idx').on( + table.capability, + table.provider, + ), + }), +); + /* ───────────────────────── CREDIT LEDGER ───────────────────────── */ export const creditLedgerEntry = pgTable( diff --git a/apps/commons-api/platform-skills/build-commons-ui-plugin/SKILL.md b/apps/commons-api/platform-skills/build-commons-ui-plugin/SKILL.md new file mode 100644 index 00000000..afff62a5 --- /dev/null +++ b/apps/commons-api/platform-skills/build-commons-ui-plugin/SKILL.md @@ -0,0 +1,27 @@ +--- +name: build-commons-ui-plugin +description: Build a sandboxed custom page or floating widget for the Commons app when a user asks for a custom UI, dashboard, control, or visualization inside Commons. +--- + +# Build a Commons UI plugin + +Create custom UI as an isolated code project. Never edit the Commons host UI or +try to reach into its DOM. + +1. Clarify the smallest useful page, widget, or both from the user's request. +2. Use `createCodeProject` and `writeCodeProjectFiles` to build an accessible, + responsive React interface that follows the Commons visual language. +3. Use `publishCodeProject`, then `testCodeProject`. Fix failures before moving + on. +4. Call `registerUiPlugin` with the published project. Request only the minimum + permissions needed: + - `theme.read` receives the current light/dark theme. + - `navigation` may ask the host to navigate to a safe internal path. +5. Tell the user the plugin is a draft and must be reviewed and enabled in + Studio Apps. + +The plugin can post `{ type: "commons:ready" }` to its parent. The host replies +with `{ type: "commons:context", theme, pluginId }`. With the `navigation` +permission, post `{ type: "commons:navigate", path: "/safe/internal/path" }`. +Never request secrets, authentication tokens, raw cookies, or unrestricted host +APIs. diff --git a/apps/commons-api/src/agent/agent.service.ts b/apps/commons-api/src/agent/agent.service.ts index e06dd15d..1ab850f6 100644 --- a/apps/commons-api/src/agent/agent.service.ts +++ b/apps/commons-api/src/agent/agent.service.ts @@ -334,7 +334,9 @@ export class AgentService implements OnModuleInit { const httpc: any = (gotMod as any).default || gotMod; // Prefer search endpoint when query provided; otherwise list voices const url = q - ? `https://api.elevenlabs.io/v1/voices/search?query=${encodeURIComponent(q)}` + ? `https://api.elevenlabs.io/v1/voices/search?query=${encodeURIComponent( + q, + )}` : `https://api.elevenlabs.io/v1/voices`; const res = await httpc.get(url, { headers: { 'xi-api-key': apiKey }, @@ -475,7 +477,11 @@ export class AgentService implements OnModuleInit { ### Agent-to-agent interaction - **interactWithAgent** — send a message to another agent and get a response. Pass the returned sessionId to continue the same conversation across calls. - - To coordinate groups of agents, use **Spaces** (see below).${childSessionsInfo ? '' : '\n - You currently have no active agent conversations.'} + - To coordinate groups of agents, use **Spaces** (see below).${ + childSessionsInfo + ? '' + : '\n - You currently have no active agent conversations.' + } ### Spaces (multi-agent collaboration) Spaces are shared channels where multiple agents and humans can communicate. @@ -524,6 +530,7 @@ export class AgentService implements OnModuleInit { For React prototypes, landing pages, dashboards, and other static frontend experiences, use lightweight code projects first. They do not require a computer and publish to durable low-cost public URLs. - **createCodeProject** — create a React project with initial files. **writeCodeProjectFiles** — write complete files directly; never squeeze source code into shell commands. - **readCodeProject** — inspect the current files and latest deployment. **publishCodeProject** — compile and publish the project. + - **registerUiPlugin** — register a published project as a sandboxed Commons page/widget draft for the user to review and enable. - **testCodeProject** — run desktop/mobile Chromium checks, inspect runtime/console/network failures, and test important interactions. A successful build is not enough: test it, fix every reported error, republish, and re-test before saying it works. - **exportCodeProjectToComputer** — move the project into the persistent computer when the work needs a backend, arbitrary packages, repository operations, ML/GPU compute, or unrestricted tooling. - Lightweight projects support React, CSS, local modules/assets, lucide-react, framer-motion, recharts, clsx, and tailwind-merge. They do not execute a Next.js server or arbitrary build plugins. @@ -593,7 +600,14 @@ export class AgentService implements OnModuleInit { ]); const childSessionsInfo = childSessions.length > 0 - ? `\n\nEXISTING CHILD SESSIONS:\nYou have the following ongoing conversations with other agents. Use these sessionIds to continue existing conversations instead of starting new ones:\n${childSessions.map((cs) => `- Agent ${cs.childAgentId}: ${cs.title || 'Untitled conversation'} (sessionId=${cs.childSessionId}, started: ${cs.createdAt})`).join('\n')}` + ? `\n\nEXISTING CHILD SESSIONS:\nYou have the following ongoing conversations with other agents. Use these sessionIds to continue existing conversations instead of starting new ones:\n${childSessions + .map( + (cs) => + `- Agent ${cs.childAgentId}: ${ + cs.title || 'Untitled conversation' + } (sessionId=${cs.childSessionId}, started: ${cs.createdAt})`, + ) + .join('\n')}` : ''; const messages: ChatCompletionMessageParam[] = [ @@ -1069,7 +1083,10 @@ export class AgentService implements OnModuleInit { computerPreparationBlock = [ '## COMPUTER PREPARATION', 'The user selected Agent Computer for this turn, but the runtime could not be prepared.', - `Reason: ${started?.errorMessage ?? 'Unknown computer provisioning error'}`, + `Reason: ${ + started?.errorMessage ?? + 'Unknown computer provisioning error' + }`, 'Explain the limitation and continue without claiming computer access. Do not call computer tools again this turn unless the user changes the computer settings.', ].join('\n'); } @@ -1326,7 +1343,7 @@ export class AgentService implements OnModuleInit { : null; const cliToolSchemas: ChatCompletionTool[] = props.cliContext - ? (dynamicCliTools ?? [ + ? dynamicCliTools ?? [ { type: 'function', function: { @@ -1521,7 +1538,7 @@ export class AgentService implements OnModuleInit { }, }, }, - ]) + ] : []; const llmWithTools = (llm as any).bindTools( @@ -1700,7 +1717,9 @@ export class AgentService implements OnModuleInit { const timer = setTimeout(() => { cleanup(); resolve( - `Error: CLI tool timed out after ${CLI_TOOL_TIMEOUT_MS / 1_000}s`, + `Error: CLI tool timed out after ${ + CLI_TOOL_TIMEOUT_MS / 1_000 + }s`, ); }, CLI_TOOL_TIMEOUT_MS); @@ -1975,7 +1994,16 @@ export class AgentService implements OnModuleInit { ]); const childSessionsInfo = childSessions.length > 0 - ? `\n\nEXISTING CHILD SESSIONS:\nYou have the following ongoing conversations with other agents. Use these sessionIds to continue existing conversations instead of starting new ones:\n${childSessions.map((cs) => `- Agent ${cs.childAgentId}: ${cs.title || 'Untitled conversation'} (sessionId=${cs.childSessionId}, started: ${cs.createdAt})`).join('\n')}` + ? `\n\nEXISTING CHILD SESSIONS:\nYou have the following ongoing conversations with other agents. Use these sessionIds to continue existing conversations instead of starting new ones:\n${childSessions + .map( + (cs) => + `- Agent ${cs.childAgentId}: ${ + cs.title || 'Untitled conversation' + } (sessionId=${cs.childSessionId}, started: ${ + cs.createdAt + })`, + ) + .join('\n')}` : ''; messages.push({ @@ -2014,7 +2042,9 @@ export class AgentService implements OnModuleInit { role: 'system', content: ` You are currently in the following space: - - Space ${space.spaceId}: ${space.name || 'Untitled space'} (created: ${space.createdAt}) + - Space ${space.spaceId}: ${ + space.name || 'Untitled space' + } (created: ${space.createdAt}) Remember your agent Id is : ${agentId} You are receiving this message because you are subscribed to this space. @@ -2122,13 +2152,13 @@ export class AgentService implements OnModuleInit { const computerSelectionDetail = computerUnavailable ? 'Computer runtime is unavailable for this turn. Do not call computer tools.' : computerRequest?.computerIds?.length - ? [ - `Assigned computer ID: ${computerRequest.computerIds[0]}`, - "This is the agent's one persistent computer and is available through the computer tools. Continue work in its existing workspace across chat sessions.", - 'Computer tool calls may omit computerId; if supplied, use the assigned ID above.', - 'Do not tell the user you lack computer access while this assigned computer is ready. If the tool fails, report that failure with evidence.', - ].join('\n') - : 'The assigned computer is not active. Call startAgentComputer before computer-backed work.'; + ? [ + `Assigned computer ID: ${computerRequest.computerIds[0]}`, + "This is the agent's one persistent computer and is available through the computer tools. Continue work in its existing workspace across chat sessions.", + 'Computer tool calls may omit computerId; if supplied, use the assigned ID above.', + 'Do not tell the user you lack computer access while this assigned computer is ready. If the tool fails, report that failure with evidence.', + ].join('\n') + : 'The assigned computer is not active. Call startAgentComputer before computer-backed work.'; const computerSelectionBlock = computerRequest?.enabled ? [ '## USER COMPUTER SELECTION', @@ -2245,7 +2275,11 @@ export class AgentService implements OnModuleInit { messages.push({ type: 'user', role: 'user', - content: `##TASK_INSTRUCTION[${nextTask.taskId}]: ${nextTask.title}\n\n${nextTask.description ?? ''}${taskContextStr}${taskToolsStr}${taskToolInstructionsStr}`, + content: `##TASK_INSTRUCTION[${nextTask.taskId}]: ${ + nextTask.title + }\n\n${ + nextTask.description ?? '' + }${taskContextStr}${taskToolsStr}${taskToolInstructionsStr}`, } as any); } @@ -2289,8 +2323,8 @@ export class AgentService implements OnModuleInit { typeof lastAi?.content === 'string' ? lastAi.content : lastAi?.content - ? JSON.stringify(lastAi.content) - : 'Task completed'; + ? JSON.stringify(lastAi.content) + : 'Task completed'; await this.db .update(schema.task) .set({ @@ -2378,10 +2412,10 @@ export class AgentService implements OnModuleInit { typeof last.content === 'string' ? last.content : typeof last === 'object' && 'content' in last - ? compact( - map((last as any).content, (_) => get(_, 'text')), - ).join('\n') - : ''; + ? compact(map((last as any).content, (_) => get(_, 'text'))).join( + '\n', + ) + : ''; const lastMessage = finalResult?.messages?.at(-1)?.toDict() ?? {}; const firstUserMessage = props.messages?.find( (m) => m.role === 'user', @@ -2989,13 +3023,22 @@ export class AgentService implements OnModuleInit { }) .where(eq(schema.agent.agentId, existing.agentId)) .returning(); - return updated ?? existing; + const resolved = updated ?? existing; + await this.skillService.ensurePlatformSkillsForCopilot( + resolved.agentId, + userId, + ); + return resolved; } + await this.skillService.ensurePlatformSkillsForCopilot( + existing.agentId, + userId, + ); return existing; } try { - return await this.createAgent({ + const created = await this.createAgent({ value: { name: 'Commons Copilot', owner: userId, @@ -3016,13 +3059,25 @@ export class AgentService implements OnModuleInit { modelId: 'gpt-5.4-mini', }, }); + await this.skillService.ensurePlatformSkillsForCopilot( + created.agentId, + userId, + ); + return created; } catch (error: any) { // Concurrent first requests may race; the partial unique index makes the // loser harmless, so return the row created by the winner. if (error?.code === '23505') { - return this.db.query.agent.findFirst({ + const created = await this.db.query.agent.findFirst({ where: (t) => and(eq(t.ownerUserId, userId), eq(t.isDefault, true)), }); + if (created) { + await this.skillService.ensurePlatformSkillsForCopilot( + created.agentId, + userId, + ); + } + return created; } throw error; } diff --git a/apps/commons-api/src/app.module.ts b/apps/commons-api/src/app.module.ts index 7c36b2df..cb45c5fb 100644 --- a/apps/commons-api/src/app.module.ts +++ b/apps/commons-api/src/app.module.ts @@ -30,6 +30,8 @@ import { FilesModule } from './files'; import { ComputerModule } from './computer'; import { AudioModule } from './audio'; import { CodeProjectModule } from './code-project'; +import { CapabilityProviderModule } from './provider'; +import { UiPluginModule } from './ui-plugin'; @Module({ imports: [ @@ -58,6 +60,8 @@ import { CodeProjectModule } from './code-project'; FilesModule, ComputerModule, CodeProjectModule, + CapabilityProviderModule, + UiPluginModule, AudioModule, MemoryModule, WalletModule, diff --git a/apps/commons-api/src/computer/computer.module.ts b/apps/commons-api/src/computer/computer.module.ts index c88074d8..657071db 100644 --- a/apps/commons-api/src/computer/computer.module.ts +++ b/apps/commons-api/src/computer/computer.module.ts @@ -6,9 +6,10 @@ import { ComputerController } from './computer.controller'; import { ComputerMigrationService } from './computer-migration.service'; import { ComputerService } from './computer.service'; import { ComputeMeteringService } from './compute-metering.service'; +import { CapabilityProviderModule } from '~/provider'; @Module({ - imports: [CreditModule, BillingModule], + imports: [CreditModule, BillingModule, CapabilityProviderModule], controllers: [ComputerController], providers: [ ComputerMigrationService, diff --git a/apps/commons-api/src/computer/computer.service.spec.ts b/apps/commons-api/src/computer/computer.service.spec.ts index dc945c8d..c14cc9e9 100644 --- a/apps/commons-api/src/computer/computer.service.spec.ts +++ b/apps/commons-api/src/computer/computer.service.spec.ts @@ -27,6 +27,7 @@ describe('ComputerService', () => { { decrypt: jest.fn() } as any, { getEntitlements: jest.fn() } as any, { getBalance: jest.fn() } as any, + { resolve: jest.fn().mockResolvedValue(null) } as any, ); jest.spyOn(service as any, 'assertCapability').mockResolvedValue({ enabled: true, @@ -254,7 +255,9 @@ describe('ComputerService', () => { resourceMode: 'elastic', storageLimit: '20Gi', } as any); - jest.spyOn(service as any, 'assertComputerSlot').mockResolvedValue(undefined); + jest + .spyOn(service as any, 'assertComputerSlot') + .mockResolvedValue(undefined); jest.spyOn(service, 'getAssignedComputer').mockResolvedValue(null as any); const returning = jest diff --git a/apps/commons-api/src/computer/computer.service.ts b/apps/commons-api/src/computer/computer.service.ts index a591e0bb..60b4eeaa 100644 --- a/apps/commons-api/src/computer/computer.service.ts +++ b/apps/commons-api/src/computer/computer.service.ts @@ -20,6 +20,7 @@ import { agentRunProgress, type AgentRunProgressEvent, } from '~/agent/run-progress'; +import { CapabilityProviderService } from '~/provider'; /** @deprecated Input compatibility only. All assigned computers are persistent. */ type LegacyComputerLifecycle = 'persistent' | 'ephemeral'; @@ -169,6 +170,7 @@ export class ComputerService { private readonly encryption: EncryptionService, private readonly entitlements: EntitlementsService, private readonly credits: CreditService, + private readonly capabilityProviders: CapabilityProviderService, ) {} /** @@ -318,8 +320,8 @@ export class ComputerService { desiredState: !config.enabled ? ('disabled' as const) : sleeping - ? ('sleeping' as const) - : ('running' as const), + ? ('sleeping' as const) + : ('running' as const), status: sleeping ? ('sleeping' as const) : computer.status, resources: this.publicResources(computer), runtimeId: computer.commonOsAgentId, @@ -612,7 +614,9 @@ export class ComputerService { stage: 'computer', status: 'running', message: 'Booting agent computer', - detail: `Persistent workspace - ${this.formatElapsed(Date.now() - now.getTime())}`, + detail: `Persistent workspace - ${this.formatElapsed( + Date.now() - now.getTime(), + )}`, payload: { progressId: computerId, computerId, @@ -703,7 +707,9 @@ export class ComputerService { 'PATCH', `/computers/${computer.commonOsAgentId}`, this.commonOsFleetId() - ? `/fleets/${this.commonOsFleetId()}/agents/${computer.commonOsAgentId}` + ? `/fleets/${this.commonOsFleetId()}/agents/${ + computer.commonOsAgentId + }` : undefined, { desiredState: 'stopped' }, args.agentId, @@ -839,7 +845,9 @@ export class ComputerService { podName: commonOs.pod?.podName ?? (commonOs.pod?.namespaceId - ? `computer-${String(commonOs._id ?? computer.commonOsAgentId).replace(/_/g, '-')}` + ? `computer-${String( + commonOs._id ?? computer.commonOsAgentId, + ).replace(/_/g, '-')}` : computer.podName), resourceProfile: commonOs.resources?.profile ?? computer.resourceProfile, @@ -871,7 +879,7 @@ export class ComputerService { : computer.startedAt, errorMessage: activeRuntimeIsStale ? 'The agent computer stopped responding and will be recovered when it is next started.' - : (commonOs.pod?.lastError ?? null), + : commonOs.pod?.lastError ?? null, updatedAt: new Date(), }) .where(eq(schema.agentComputerInstance.computerId, computerId)) @@ -916,9 +924,13 @@ export class ComputerService { const path = normalizeWorkspacePath(args.path); const data = await this.commonOsComputerRequest<{ content?: string }>( 'GET', - `/computers/${computer.commonOsAgentId}/workspace/read?path=${encodeURIComponent(path)}`, + `/computers/${ + computer.commonOsAgentId + }/workspace/read?path=${encodeURIComponent(path)}`, this.commonOsFleetId() - ? `/fleets/${this.commonOsFleetId()}/agents/${computer.commonOsAgentId}/workspace/read?path=${encodeURIComponent(path)}` + ? `/fleets/${this.commonOsFleetId()}/agents/${ + computer.commonOsAgentId + }/workspace/read?path=${encodeURIComponent(path)}` : undefined, undefined, computer.agentId, @@ -1042,7 +1054,9 @@ export class ComputerService { stage, status: 'running', message: this.submittingMessageForComputerEvent(args.eventType), - detail: `${args.summary} - ${this.formatElapsed(Date.now() - submitStartedAt)}`, + detail: `${args.summary} - ${this.formatElapsed( + Date.now() - submitStartedAt, + )}`, payload: { progressId, computerId: computer.computerId, @@ -1059,7 +1073,9 @@ export class ComputerService { 'POST', `/computers/${computer.commonOsAgentId}/instructions`, this.commonOsFleetId() - ? `/fleets/${this.commonOsFleetId()}/agents/${computer.commonOsAgentId}/human-message` + ? `/fleets/${this.commonOsFleetId()}/agents/${ + computer.commonOsAgentId + }/human-message` : undefined, { content: args.instruction, @@ -1131,7 +1147,10 @@ export class ComputerService { stage, status: 'running', message: this.waitingMessageForComputerEvent(args.eventType), - detail: `${args.summary} - ${(progress.status ?? 'pending').replace(/_/g, ' ')} - ${this.formatElapsed(progress.elapsedMs)}`, + detail: `${args.summary} - ${(progress.status ?? 'pending').replace( + /_/g, + ' ', + )} - ${this.formatElapsed(progress.elapsedMs)}`, payload: { progressId, computerId: computer.computerId, @@ -1347,7 +1366,11 @@ export class ComputerService { args.url ? `Open ${args.url} first.` : 'Use the currently open page.', 'Wait until the page is interactive. Use browser_inspect and browser_eval to inspect rendered text, the application root, runtime overlays, console errors, page errors, and failed requests.', actions.length - ? `Perform these actions in order and verify their effects:\n${JSON.stringify(actions, null, 2)}` + ? `Perform these actions in order and verify their effects:\n${JSON.stringify( + actions, + null, + 2, + )}` : 'Exercise the primary visible interaction when it is safe and reversible.', 'Capture a final browser screenshot.', 'Return a concise test report with passed checks, failed checks, current URL/title, diagnostics, and screenshot status.', @@ -1391,13 +1414,17 @@ export class ComputerService { }).catch(() => []); const computer = computers[0]; const line = computer - ? `- ${computer.name} (${computer.computerId}) persistent/${computer.status}${computer.browser?.url ? ` browser=${computer.browser.url}` : ''}` + ? `- ${computer.name} (${computer.computerId}) persistent/${ + computer.status + }${computer.browser?.url ? ` browser=${computer.browser.url}` : ''}` : '- The persistent computer has not been provisioned yet.'; return [ '### Persistent computer', 'This agent can have exactly one isolated CommonOS computer. Its workspace persists across chats and pod restarts.', - `Computer use is ${config.allowAgentStart ? 'agent-wakeable' : 'user-wake only'}; resource profile is ${config.resourceProfile}/${config.resourceMode}.`, + `Computer use is ${ + config.allowAgentStart ? 'agent-wakeable' : 'user-wake only' + }; resource profile is ${config.resourceProfile}/${config.resourceMode}.`, 'Use startAgentComputer to wake or attach the assigned computer before computer work. It is idempotent and never creates an extra computer. Use writeComputerFiles for complete source files, runComputerCommand for finite terminal work, readComputerFile for files, and testComputerBrowser for application verification.', 'Never encode source files in shell heredocs or long commands. writeComputerFiles is the reliable structured file-writing path.', 'Always keep commands scoped to the task, avoid secrets exfiltration, and summarize created files/screenshots/results for the user.', @@ -1547,9 +1574,9 @@ export class ComputerService { computer: ComputerInstance; name: string; }) { - if (!this.commonOsConfigured()) { + if (!(await this.computerProviderConfigured(args.agent))) { throw new Error( - 'CommonOS computer API is not configured. Set COMMON_OS_API_URL and COMMON_OS_API_KEY.', + 'The selected agent computer provider is not configured.', ); } @@ -1558,8 +1585,8 @@ export class ComputerService { runtimeType === 'openclaw' || runtimeType === 'hermes' ? runtimeType : runtimeType === 'custom' - ? 'guest' - : 'native'; + ? 'guest' + : 'native'; const modelApiKey = this.decryptStoredSecret(args.agent.modelApiKey); const runtimeConfig = (args.agent.runtimeConfig ?? {}) as Record< string, @@ -1831,7 +1858,9 @@ export class ComputerService { if (this.commonOsUseGeneralComputerApi()) { if (snapshotSupported) { const suffix = lastEventAt - ? `?after=${encodeURIComponent(lastEventAt)}${lastEventId ? `&afterId=${encodeURIComponent(lastEventId)}` : ''}` + ? `?after=${encodeURIComponent(lastEventAt)}${ + lastEventId ? `&afterId=${encodeURIComponent(lastEventId)}` : '' + }` : ''; try { const snapshot = await this.commonOsComputerRequest<{ @@ -1849,7 +1878,9 @@ export class ComputerService { } catch (error) { snapshotSupported = false; this.logger.warn( - `CommonOS instruction snapshot is unavailable; using compatibility polling: ${error instanceof Error ? error.message : String(error)}`, + `CommonOS instruction snapshot is unavailable; using compatibility polling: ${ + error instanceof Error ? error.message : String(error) + }`, ); } } @@ -1939,7 +1970,9 @@ export class ComputerService { } catch (err) { if (!legacyFleetPath) throw err; this.logger.warn( - `CommonOS computer API ${method} ${computerPath} failed; falling back to fleet route: ${err instanceof Error ? err.message : String(err)}`, + `CommonOS computer API ${method} ${computerPath} failed; falling back to fleet route: ${ + err instanceof Error ? err.message : String(err) + }`, ); } } @@ -1962,12 +1995,32 @@ export class ComputerService { body?: unknown, agentCommonsId?: string, ): Promise { - const apiUrl = this.commonOsApiUrl(); - const apiKey = this.commonOsApiKey(); + let apiUrl = this.commonOsApiUrl(); + let apiKey = this.commonOsApiKey(); + let basePath = this.commonOsBasePath(apiUrl); + if (agentCommonsId) { + const agent = await this.db.query.agent.findFirst({ + where: (table) => eq(table.agentId, agentCommonsId), + columns: { ownerUserId: true, owner: true }, + }); + const ownerId = agent?.ownerUserId ?? agent?.owner; + const configured = ownerId + ? await this.capabilityProviders.resolve(ownerId, 'computer') + : null; + if (configured?.provider === 'custom') { + apiUrl = configured.endpointUrl?.replace(/\/$/, '') ?? ''; + apiKey = configured.credentials.apiKey ?? ''; + basePath = String(configured.settings.basePath ?? ''); + } else if (configured && configured.provider !== 'commonos') { + throw new Error( + `Computer provider "${configured.provider}" has no installed adapter`, + ); + } + } if (!apiUrl || !apiKey) { - throw new Error('CommonOS API credentials are not configured'); + throw new Error('Agent computer API credentials are not configured'); } - const url = `${apiUrl}${this.commonOsBasePath(apiUrl)}${path}`; + const url = `${apiUrl}${basePath}${path}`; const response = await fetch(url, { method, headers: { @@ -2166,7 +2219,7 @@ export class ComputerService { } normalized.gpuCount = count; normalized.gpuType = - count > 0 ? (resources.gpu?.type ?? 'nvidia-l4') : null; + count > 0 ? resources.gpu?.type ?? 'nvidia-l4' : null; } } @@ -2341,8 +2394,8 @@ export class ComputerService { capability === 'browser' ? config.allowBrowser : capability === 'terminal' - ? config.allowTerminal - : config.allowFilesystem; + ? config.allowTerminal + : config.allowFilesystem; if (!allowed) { throw new BadRequestException( `${capability} access is disabled for this computer`, @@ -2428,6 +2481,19 @@ export class ComputerService { ); } + private async computerProviderConfigured( + agent: typeof schema.agent.$inferSelect, + ) { + const ownerId = agent.ownerUserId ?? agent.owner; + const configured = ownerId + ? await this.capabilityProviders.resolve(ownerId, 'computer') + : null; + if (configured?.provider === 'custom') { + return Boolean(configured.endpointUrl && configured.credentials.apiKey); + } + return this.commonOsConfigured(); + } + private commonOsApiUrl() { return ( process.env.COMMON_OS_API_URL || diff --git a/apps/commons-api/src/files/files.service.spec.ts b/apps/commons-api/src/files/files.service.spec.ts index ad2a1b4c..fcd45943 100644 --- a/apps/commons-api/src/files/files.service.spec.ts +++ b/apps/commons-api/src/files/files.service.spec.ts @@ -11,6 +11,80 @@ import { describe('FilesService document support', () => { const service = new FilesService({} as any, {} as any, {} as any); + it('reuses an identical Library upload and links it to the current agent session', async () => { + const existing = { + itemId: 'existing-file', + name: 'brief.pdf', + mimeType: 'application/pdf', + kind: 'pdf', + sizeBytes: 12, + status: 'ready', + textPreview: 'Existing content', + extractedTextChars: 16, + metadata: { storageProvider: 's3' }, + ownerUserId: 'user-1', + workspaceId: null, + sourceAgentId: 'agent-previous', + sourceSessionId: 'session-previous', + sha256: 'hash', + source: 'upload', + visibility: 'private', + isFavorite: false, + deletedAt: null, + extractionError: null, + createdAt: new Date(), + updatedAt: new Date(), + }; + const db = { + query: { + libraryItem: { findFirst: jest.fn().mockResolvedValue(existing) }, + }, + }; + const uploadService = new FilesService(db as any, {} as any, {} as any); + jest + .spyOn(uploadService as any, 'resolveOwnership') + .mockResolvedValue({ ownerUserId: 'user-1', workspaceId: null }); + jest + .spyOn(uploadService as any, 'resolveStorageProvider') + .mockResolvedValue('s3'); + jest.spyOn(uploadService as any, 'getArtifacts').mockResolvedValue([]); + const link = jest + .spyOn(uploadService as any, 'linkArtifactScopes') + .mockResolvedValue(undefined); + const audit = jest + .spyOn(uploadService as any, 'audit') + .mockResolvedValue(undefined); + const store = jest.spyOn(uploadService as any, 'storeBuffer'); + + const result = await (uploadService as any).persistFile({ + buffer: Buffer.from('same content'), + originalName: 'brief-copy.pdf', + mimeType: 'application/pdf', + ownerId: 'user-1', + ownerType: 'user', + agentId: 'agent-1', + sessionId: 'session-1', + deduplicate: true, + }); + + expect(result).toMatchObject({ fileId: 'existing-file', reused: true }); + expect(link).toHaveBeenCalledWith( + 'existing-file', + expect.objectContaining({ + agentId: 'agent-1', + sessionId: 'session-1', + }), + ); + expect(audit).toHaveBeenCalledWith( + 'existing-file', + 'user', + 'user-1', + 'reused', + expect.objectContaining({ uploadedName: 'brief-copy.pdf' }), + ); + expect(store).not.toHaveBeenCalled(); + }); + it('returns structured and SDK-compatible URLs for uploaded images', async () => { const imageService = new FilesService({} as any, {} as any, {} as any); jest.spyOn(imageService as any, 'getFileOrThrow').mockResolvedValue({ @@ -22,7 +96,9 @@ describe('FilesService document support', () => { textPreview: '', metadata: {}, }); - jest.spyOn(imageService as any, 'assertCanAccess').mockResolvedValue(undefined); + jest + .spyOn(imageService as any, 'assertCanAccess') + .mockResolvedValue(undefined); jest.spyOn(imageService as any, 'getBlobs').mockResolvedValue([]); jest.spyOn(imageService as any, 'getArtifacts').mockResolvedValue([ { @@ -336,6 +412,11 @@ describe('FilesService document support', () => { findFirst: jest.fn(), }, }, + insert: jest.fn().mockReturnValue({ + values: jest.fn().mockReturnValue({ + onConflictDoNothing: jest.fn().mockResolvedValue(undefined), + }), + }), }; const workspaceService = new FilesService(db as any, {} as any, {} as any); diff --git a/apps/commons-api/src/files/files.service.ts b/apps/commons-api/src/files/files.service.ts index b5dbeabc..055eea6a 100644 --- a/apps/commons-api/src/files/files.service.ts +++ b/apps/commons-api/src/files/files.service.ts @@ -14,7 +14,7 @@ import { } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { awsCredentialsProvider } from '@vercel/oidc-aws-credentials-provider'; -import { and, eq, gt, inArray, isNull, or } from 'drizzle-orm'; +import { and, eq, gt, inArray, isNull, or, sql } from 'drizzle-orm'; import crypto from 'crypto'; import sharp from 'sharp'; import mammoth from 'mammoth'; @@ -22,6 +22,10 @@ import * as XLSX from 'xlsx'; import JSZip from 'jszip'; import PptxGenJS from 'pptxgenjs'; import path from 'node:path'; +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { promisify } from 'node:util'; import { PDFArray, PDFDict, @@ -68,6 +72,8 @@ export type FileAttachmentRef = { status: string; textPreview?: string | null; extractedTextChars: number; + /** True when the upload reused an existing Library item with identical bytes. */ + reused?: boolean; artifacts?: Array<{ artifactId: string; kind: string; @@ -93,6 +99,7 @@ type PersistFileInput = { extractedTextOverride?: string; additionalArtifacts?: ExtractedArtifact[]; onPersistenceStage?: (stage: string) => void; + deduplicate?: boolean; }; type ExtractedArtifact = { @@ -130,6 +137,8 @@ type LoadedPresentationImage = { height: number; }; +const execFileAsync = promisify(execFile); + export type PdfTextReplacement = { /** Exact text copied from readUploadedFile, without the page marker. */ find: string; @@ -223,6 +232,7 @@ export class FilesService { buffer: file.buffer, originalName: file.originalname, mimeType: file.mimetype, + deduplicate: true, }), ); } @@ -483,9 +493,7 @@ export class FilesService { ), ), ); - const imageSlides = slides.filter( - (slide) => slide.imageFileId, - ).length; + const imageSlides = slides.filter((slide) => slide.imageFileId).length; const notesSlides = slides.filter((slide) => Boolean(slide.notes?.trim()), ).length; @@ -505,7 +513,9 @@ export class FilesService { : []), ...(notesSlides < slides.length ? [ - `${slides.length - notesSlides} slide(s) do not include speaker notes.`, + `${ + slides.length - notesSlides + } slide(s) do not include speaker notes.`, ] : []), ], @@ -820,7 +830,7 @@ export class FilesService { extractedText.storageBucket, extractedText.storagePath, ) - : (file.textPreview ?? ''); + : file.textPreview ?? ''; const content = fullText.slice(offset, offset + maxChars); const nextOffset = offset + content.length; const artifacts = await this.getArtifacts(file.itemId); @@ -898,36 +908,14 @@ export class FilesService { const rowsById = new Map(rows.map((row) => [row.itemId, row])); const ordered = ids.map((id) => rowsById.get(id)).filter(Boolean); for (const file of ordered) await this.assertCanAccess(file!, context); - - const bindToSessionIds = context.sessionId - ? ordered - .filter( - (file) => - file && - !file.sourceSessionId && - (!file.sourceAgentId || file.sourceAgentId === context.agentId) && - (!context.ownerId || - samePrincipal(file.ownerUserId, context.ownerId)), - ) - .map((file) => file!.itemId) - : []; - if (bindToSessionIds.length) { - await this.db - .insert(schema.libraryLink) - .values( - bindToSessionIds.map((itemId) => ({ - itemId, - scopeType: 'session', - scopeId: context.sessionId!, - })), - ) - .onConflictDoNothing(); - for (const file of ordered) { - if (file && bindToSessionIds.includes(file.itemId)) { - file.sourceSessionId = context.sessionId!; - } - } - } + await Promise.all( + ordered.map((file) => + this.linkArtifactScopes(file!.itemId, { + agentId: context.agentId, + sessionId: context.sessionId, + }), + ), + ); const artifacts = ordered.length ? await this.db.query.libraryBlob.findMany({ @@ -967,7 +955,13 @@ export class FilesService { const preview = file.textPreview ? `\nPreview:\n${file.textPreview}` : ''; - return `${index + 1}. ${file.name} (fileId: ${file.fileId}, ${file.kind}, ${file.mimeType}, ${formatBytes(file.sizeBytes)}, status: ${file.status}). Extracted text chars: ${file.extractedTextChars}.${artifactSummary}${preview}`; + return `${index + 1}. ${file.name} (fileId: ${file.fileId}, ${ + file.kind + }, ${file.mimeType}, ${formatBytes(file.sizeBytes)}, status: ${ + file.status + }). Extracted text chars: ${ + file.extractedTextChars + }.${artifactSummary}${preview}`; }), ]; @@ -1095,7 +1089,6 @@ export class FilesService { ); } - const fileId = uuidv4(); const originalName = sanitizeFileName(input.originalName || 'upload'); const mimeType = normalizeMimeType(input.mimeType, originalName); const kind = classifyFile(mimeType, originalName); @@ -1110,6 +1103,41 @@ export class FilesService { ownership.ownerUserId, input.storageProvider, ); + if (input.deduplicate) { + updateStage('checking the Library for an identical file'); + const reusable = await this.db.query.libraryItem.findFirst({ + where: (table) => + and( + sql`lower(${table.ownerUserId}) = lower(${ownership.ownerUserId})`, + eq(table.sha256, sha256), + isNull(table.deletedAt), + inArray(table.status, ['ready', 'partial']), + ), + }); + if (reusable) { + await this.linkArtifactScopes(reusable.itemId, input); + await this.audit( + reusable.itemId, + input.ownerType ?? 'user', + input.ownerId ?? ownership.ownerUserId, + 'reused', + { + agentId: input.agentId ?? null, + sessionId: input.sessionId ?? null, + uploadedName: originalName, + }, + ); + return { + ...this.toAttachmentRef( + reusable, + await this.getArtifacts(reusable.itemId), + ), + reused: true, + }; + } + } + + const fileId = uuidv4(); const bucket = storageProvider === 's3' ? this.bucketName() : 'ipfs'; if (storageProvider === 's3') { updateStage('checking presentation storage'); @@ -1192,9 +1220,13 @@ export class FilesService { for (const artifact of extraction.artifacts) { updateStage( - `storing presentation preview ${artifact.pageNumber ?? persistedArtifacts.length + 1}`, + `storing presentation preview ${ + artifact.pageNumber ?? persistedArtifacts.length + 1 + }`, ); - const artifactPath = `${basePath}/derived/${sanitizeFileName(artifact.fileName)}`; + const artifactPath = `${basePath}/derived/${sanitizeFileName( + artifact.fileName, + )}`; const storedArtifact = await this.storeBuffer( storageProvider, bucket, @@ -1309,17 +1341,8 @@ export class FilesService { : []), ]); - if (input.sessionId) { - updateStage('linking the presentation to the session'); - await this.db - .insert(schema.libraryLink) - .values({ - itemId: fileId, - scopeType: 'session', - scopeId: input.sessionId, - }) - .onConflictDoNothing(); - } + updateStage('linking the presentation to its agent and session'); + await this.linkArtifactScopes(fileId, input); updateStage('indexing presentation text'); await this.indexText(fileId, text); updateStage('auditing presentation creation'); @@ -1406,7 +1429,9 @@ export class FilesService { // Always point pdf.js at its packaged fonts so rendered artifacts match // the actual PDF instead of silently dropping most glyphs. useSystemFonts: false, - standardFontDataUrl: `${path.join(pdfjsRoot, 'standard_fonts')}${path.sep}`, + standardFontDataUrl: `${path.join(pdfjsRoot, 'standard_fonts')}${ + path.sep + }`, cMapUrl: `${path.join(pdfjsRoot, 'cmaps')}${path.sep}`, cMapPacked: true, }); @@ -1819,6 +1844,25 @@ export class FilesService { mimeType: string, kind: 'audio' | 'video', ): Promise { + if ( + kind === 'video' && + process.env.AGENT_FILE_VIDEO_UNDERSTANDING_ENABLED !== 'false' && + process.env.OPENAI_API_KEY + ) { + try { + return await this.extractVideoUnderstanding( + buffer, + originalName, + mimeType, + ); + } catch (error) { + this.logger.warn( + `Video understanding failed for ${originalName}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + } if ( kind === 'audio' && process.env.AGENT_FILE_AUDIO_TRANSCRIPTION_ENABLED === 'true' && @@ -1868,6 +1912,143 @@ export class FilesService { }; } + private async extractVideoUnderstanding( + buffer: Buffer, + originalName: string, + mimeType: string, + ): Promise { + const directory = await mkdtemp(path.join(tmpdir(), 'commons-video-')); + const extension = videoExtension(mimeType, originalName); + const inputPath = path.join(directory, `input${extension}`); + try { + await writeFile(inputPath, buffer); + const framePattern = path.join(directory, 'frame-%02d.jpg'); + const maxFrames = Math.max( + 3, + Math.min(12, Number(process.env.AGENT_FILE_VIDEO_MAX_FRAMES ?? 8)), + ); + await execFileAsync( + process.env.FFMPEG_PATH || 'ffmpeg', + [ + '-hide_banner', + '-loglevel', + 'error', + '-i', + inputPath, + '-vf', + "fps=1/8,scale='min(1280,iw)':-2", + '-frames:v', + String(maxFrames), + '-q:v', + '3', + framePattern, + ], + { maxBuffer: 4 * 1024 * 1024 }, + ); + const frameNames = (await readdir(directory)) + .filter((name) => /^frame-\d+\.jpg$/.test(name)) + .sort() + .slice(0, maxFrames); + if (!frameNames.length) throw new Error('No video frames were decoded'); + + let transcript = ''; + const audioPath = path.join(directory, 'audio.mp3'); + try { + await execFileAsync( + process.env.FFMPEG_PATH || 'ffmpeg', + [ + '-hide_banner', + '-loglevel', + 'error', + '-i', + inputPath, + '-vn', + '-ac', + '1', + '-ar', + '16000', + '-b:a', + '48k', + '-y', + audioPath, + ], + { maxBuffer: 4 * 1024 * 1024 }, + ); + const audio = await readFile(audioPath); + if (audio.length > 0 && audio.length <= 25 * 1024 * 1024) { + const audioFile = new File([new Uint8Array(audio)], 'audio.mp3', { + type: 'audio/mpeg', + }); + const transcription = await this.openAI.audio.transcriptions.create({ + file: audioFile, + model: + process.env.AGENT_FILE_AUDIO_TRANSCRIPTION_MODEL || + 'gpt-4o-mini-transcribe', + }); + transcript = transcription.text?.trim() ?? ''; + } + } catch { + // Silent videos and unavailable audio codecs are valid inputs. + } + + const frames = await Promise.all( + frameNames.map(async (name) => ({ + type: 'image_url' as const, + image_url: { + url: `data:image/jpeg;base64,${( + await readFile(path.join(directory, name)) + ).toString('base64')}`, + detail: 'low' as const, + }, + })), + ); + const model = + process.env.AGENT_FILE_VIDEO_UNDERSTANDING_MODEL || 'gpt-5.4-mini'; + const completion = await this.openAI.chat.completions.create({ + model, + messages: [ + { + role: 'system', + content: + 'Analyze videos for an AI agent. Produce a concise but complete searchable description with: purpose, chronological actions/events, visible text or UI, spoken instructions, results, and uncertainties. Do not invent details between sampled frames.', + }, + { + role: 'user', + content: [ + { + type: 'text', + text: `Video: ${originalName}\nSampled frames are chronological.${ + transcript ? `\nAudio transcript:\n${transcript}` : '' + }`, + }, + ...frames, + ], + }, + ], + }); + const analysis = completion.choices[0]?.message?.content?.trim() ?? ''; + const text = [ + analysis, + transcript ? `\n## Transcript\n${transcript}` : '', + ] + .filter(Boolean) + .join('\n'); + return { + text, + metadata: { + mediaKind: 'video', + videoUnderstandingModel: model, + sampledFrames: frameNames.length, + hasTranscript: Boolean(transcript), + }, + artifacts: [], + status: text ? 'ready' : 'partial', + }; + } finally { + await rm(directory, { recursive: true, force: true }).catch(() => null); + } + } + private async imageMetadata(buffer: Buffer) { const metadata = await sharp(buffer).metadata(); return { @@ -1976,6 +2157,29 @@ export class FilesService { throw new NotFoundException('File not found'); } + private async linkArtifactScopes( + itemId: string, + input: { agentId?: string | null; sessionId?: string | null }, + ) { + const links = [ + input.agentId + ? { itemId, scopeType: 'agent', scopeId: input.agentId } + : null, + input.sessionId + ? { itemId, scopeType: 'session', scopeId: input.sessionId } + : null, + ].filter(Boolean) as Array<{ + itemId: string; + scopeType: string; + scopeId: string; + }>; + if (!links.length) return; + await this.db + .insert(schema.libraryLink) + .values(links) + .onConflictDoNothing(); + } + private async downloadText(bucket: string, path: string) { if (bucket === 'ipfs') { const data = await this.pinata.fetchFile(path); @@ -2051,8 +2255,8 @@ export class FilesService { ServerSideEncryption: process.env.AGENT_FILES_S3_KMS_KEY_ID ? 'aws:kms' : process.env.AGENT_FILE_S3_SSE === 'false' - ? undefined - : 'AES256', + ? undefined + : 'AES256', SSEKMSKeyId: process.env.AGENT_FILES_S3_KMS_KEY_ID, }), ); @@ -2200,7 +2404,9 @@ export class FilesService { private capExtractedText(text: string) { const max = this.maxExtractedTextChars(); if (text.length <= max) return text; - return `${text.slice(0, max)}\n\n[truncated: showing first ${max} of ${text.length} extracted characters]`; + return `${text.slice(0, max)}\n\n[truncated: showing first ${max} of ${ + text.length + } extracted characters]`; } private toAttachmentRef( @@ -2295,7 +2501,9 @@ export class FilesService { } } catch (error) { this.logger.warn( - `Artifact embedding deferred for ${itemId}: ${error instanceof Error ? error.message : String(error)}`, + `Artifact embedding deferred for ${itemId}: ${ + error instanceof Error ? error.message : String(error) + }`, ); } } @@ -2320,7 +2528,10 @@ export class FilesService { Bucket: original.storageBucket, Key: original.storagePath, ResponseContentType: file.mimeType, - ResponseContentDisposition: `inline; filename="${file.name.replace(/"/g, '')}"`, + ResponseContentDisposition: `inline; filename="${file.name.replace( + /"/g, + '', + )}"`, }) as any, { expiresIn: Number( @@ -2344,12 +2555,14 @@ export class FilesService { actorType: 'user' | 'agent' | 'service', actorId: string, action: string, + metadata: Record = {}, ) { await this.db.insert(schema.libraryAuditEvent).values({ itemId, actorType, actorId, action, + metadata, }); } } @@ -2801,25 +3014,43 @@ async function renderPresentationPreview( .slice(0, 2) .map( (line, lineIndex) => - `${escapeXml(line)}`, + `${escapeXml( + line, + )}`, ) .join(''); const body = wrapPreviewText(card.body, columns === 2 ? 48 : 30) .slice(0, 3) .map( (line, lineIndex) => - `${escapeXml(line)}`, + `${escapeXml( + line, + )}`, ) .join(''); return ` - - ${String(cardIndex + 1).padStart(2, '0')} - ${heading} - ${body}`; + + ${String(cardIndex + 1).padStart(2, '0')} + ${heading} + ${body}`; }) .join(''); const imageMarkup = image - ? `` + ? `` : ''; const bodyX = layout === 'image-left' ? 650 : 88; const bodyWidth = @@ -2827,22 +3058,34 @@ async function renderPresentationPreview( const svg = ` - ${escapeXml(spec.title ?? '')} + ${escapeXml(spec.title ?? '')} ${imageMarkup} ${cardMarkup} ${ cards.length ? '' - : ` + : ` ${bodyLines .map( (line, lineIndex) => - `${escapeXml(line.slice(0, bodyWidth / 14))}`, + `${escapeXml( + line.slice(0, bodyWidth / 14), + )}`, ) .join('')} ` } - ${String(index + 1).padStart(2, '0')} + ${String(index + 1).padStart(2, '0')} `; buffer = await sharp(Buffer.from(svg)).png().toBuffer(); } @@ -2965,6 +3208,17 @@ function sanitizePathSegment(value: string) { ); } +function videoExtension(mimeType: string, originalName: string) { + const lower = originalName.toLowerCase(); + const known = ['.mp4', '.m4v', '.mov', '.webm', '.avi']; + const fromName = known.find((extension) => lower.endsWith(extension)); + if (fromName) return fromName; + if (mimeType.includes('quicktime')) return '.mov'; + if (mimeType.includes('webm')) return '.webm'; + if (mimeType.includes('avi')) return '.avi'; + return '.mp4'; +} + export function normalizeMimeType( mimeType: string | undefined, fileName: string, @@ -3155,7 +3409,9 @@ export async function revisePdfBufferPreservingLayout( data: new Uint8Array(sourceBuffer), disableWorker: true, useSystemFonts: false, - standardFontDataUrl: `${path.join(pdfjsRoot, 'standard_fonts')}${path.sep}`, + standardFontDataUrl: `${path.join(pdfjsRoot, 'standard_fonts')}${ + path.sep + }`, cMapUrl: `${path.join(pdfjsRoot, 'cmaps')}${path.sep}`, cMapPacked: true, }); @@ -3201,7 +3457,9 @@ export async function revisePdfBufferPreservingLayout( ); if (!selectedItems.length) { throw new BadRequestException( - `Could not map PDF text "${previewErrorText(find)}" to a visible text region`, + `Could not map PDF text "${previewErrorText( + find, + )}" to a visible text region`, ); } @@ -3218,7 +3476,9 @@ export async function revisePdfBufferPreservingLayout( const fontNames = new Set(selectedItems.map((item) => item.fontName)); if (fontNames.size !== 1) { throw new BadRequestException( - `The passage "${previewErrorText(find)}" crosses differently styled text. Split it into one replacement per style.`, + `The passage "${previewErrorText( + find, + )}" crosses differently styled text. Split it into one replacement per style.`, ); } if ( @@ -3262,7 +3522,9 @@ export async function revisePdfBufferPreservingLayout( ) ) { throw new BadRequestException( - `The passage "${previewErrorText(find)}" mixes font sizes. Split it into smaller replacements.`, + `The passage "${previewErrorText( + find, + )}" mixes font sizes. Split it into smaller replacements.`, ); } const minX = Math.min(...lines.map((line) => line.minX)); @@ -3276,7 +3538,11 @@ export async function revisePdfBufferPreservingLayout( ); if (wrapped.length > lines.length) { throw new BadRequestException( - `The replacement for "${previewErrorText(find)}" needs ${wrapped.length} lines but the original region has ${lines.length}. Shorten the replacement or split the edit.`, + `The replacement for "${previewErrorText(find)}" needs ${ + wrapped.length + } lines but the original region has ${ + lines.length + }. Shorten the replacement or split the edit.`, ); } @@ -3385,7 +3651,9 @@ function findPdfTextMatch( } } throw new BadRequestException( - `Could not find occurrence ${occurrence} of "${previewErrorText(find)}" in the source PDF. Copy the exact passage from readUploadedFile.`, + `Could not find occurrence ${occurrence} of "${previewErrorText( + find, + )}" in the source PDF. Copy the exact passage from readUploadedFile.`, ); } @@ -3477,7 +3745,11 @@ function assertPdfFontSupportsText( } if (missing.size) { throw new BadRequestException( - `The source PDF embeds a subset of ${fontName} without these replacement glyphs: ${[...missing].join(' ')}. Rephrase the replacement using characters already present in the document so the original font can be preserved.`, + `The source PDF embeds a subset of ${fontName} without these replacement glyphs: ${[ + ...missing, + ].join( + ' ', + )}. Rephrase the replacement using characters already present in the document so the original font can be preserved.`, ); } } @@ -3533,7 +3805,9 @@ function wrapTextToPdfWidth( for (const word of words) { if (widthOfTextAtSize(word, fontSize) > maxWidth) { throw new BadRequestException( - `The word "${previewErrorText(word)}" is wider than the original PDF text region`, + `The word "${previewErrorText( + word, + )}" is wider than the original PDF text region`, ); } const candidate = current ? `${current} ${word}` : word; @@ -3780,10 +4054,16 @@ function docxParagraph(text: string, style?: string) { const runs = lines .map( (line, index) => - `${index ? '' : ''}${escapeXml(line)}`, + `${ + index ? '' : '' + }${escapeXml(line)}`, ) .join(''); - return `${style ? `` : ''}${runs}`; + return `${ + style + ? `` + : '' + }${runs}`; } function escapeXml(value: string) { diff --git a/apps/commons-api/src/files/library.controller.ts b/apps/commons-api/src/files/library.controller.ts index cf71b8fb..84cc784f 100644 --- a/apps/commons-api/src/files/library.controller.ts +++ b/apps/commons-api/src/files/library.controller.ts @@ -46,6 +46,7 @@ export class LibraryController { @Query('source') source?: string, @Query('favorite') favorite?: string, @Query('sessionId') sessionId?: string, + @Query('agentId') agentId?: string, @Query('limit') limit?: string, @Query('offset') offset?: string, ) { @@ -55,6 +56,7 @@ export class LibraryController { source, favorite: favorite === 'true', sessionId, + agentId, limit: limit ? Number(limit) : undefined, offset: offset ? Number(offset) : undefined, }); diff --git a/apps/commons-api/src/files/library.service.ts b/apps/commons-api/src/files/library.service.ts index 024431e7..e8b6be34 100644 --- a/apps/commons-api/src/files/library.service.ts +++ b/apps/commons-api/src/files/library.service.ts @@ -45,6 +45,7 @@ export class LibraryService { source?: string; favorite?: boolean; sessionId?: string; + agentId?: string; limit?: number; offset?: number; }, @@ -69,7 +70,25 @@ export class LibraryService { : undefined, filters.favorite ? eq(schema.libraryItem.isFavorite, true) : undefined, filters.sessionId - ? eq(schema.libraryItem.sourceSessionId, filters.sessionId) + ? or( + eq(schema.libraryItem.sourceSessionId, filters.sessionId), + this.linkedToScope('session', filters.sessionId), + ) + : undefined, + filters.agentId + ? or( + eq(schema.libraryItem.sourceAgentId, filters.agentId), + this.linkedToScope('agent', filters.agentId), + sql`EXISTS ( + SELECT 1 + FROM library_link agent_session_link + INNER JOIN session linked_session + ON linked_session.session_id::text = agent_session_link.scope_id + WHERE agent_session_link.item_id = ${schema.libraryItem.itemId} + AND agent_session_link.scope_type = 'session' + AND linked_session.agent_id = ${filters.agentId} + )`, + ) : undefined, ]; const items = await this.db.query.libraryItem.findMany({ @@ -92,7 +111,9 @@ export class LibraryService { return Promise.all( items.map(async (item) => ({ ...this.publicItem(item), - sessionTitle: item.sourceSessionId + sessionTitle: + item.sourceSessionId && + (!filters.agentId || item.sourceAgentId === filters.agentId) ? (titleBySession.get(item.sourceSessionId) ?? 'Untitled chat') : null, previewUrl: @@ -540,6 +561,15 @@ export class LibraryService { return undefined; } + private linkedToScope(scopeType: 'agent' | 'session', scopeId: string) { + return sql`EXISTS ( + SELECT 1 FROM library_link scoped_link + WHERE scoped_link.item_id = ${schema.libraryItem.itemId} + AND scoped_link.scope_type = ${scopeType} + AND scoped_link.scope_id = ${scopeId} + )`; + } + private async getAccessible(itemId: string, principal: LibraryPrincipal) { const item = await this.db.query.libraryItem.findFirst({ where: (table) => diff --git a/apps/commons-api/src/provider/capability-provider.controller.ts b/apps/commons-api/src/provider/capability-provider.controller.ts new file mode 100644 index 00000000..b8878d75 --- /dev/null +++ b/apps/commons-api/src/provider/capability-provider.controller.ts @@ -0,0 +1,51 @@ +import { Body, Controller, Delete, Get, Param, Put, Req } from '@nestjs/common'; +import type { Request } from 'express'; +import { resolveCallerId, type ApiKeyPrincipal } from '~/modules/auth'; +import { + CapabilityProviderInput, + CapabilityProviderService, + type CapabilityName, +} from './capability-provider.service'; + +@Controller({ version: '1', path: 'providers' }) +export class CapabilityProviderController { + constructor(private readonly providers: CapabilityProviderService) {} + + @Get() + list(@Req() request: Request) { + const principal = requester(request); + return this.providers.list(principal.principalId); + } + + @Put(':capability') + async upsert( + @Req() request: Request, + @Param('capability') capability: CapabilityName, + @Body() body: CapabilityProviderInput, + ) { + const principal = requester(request); + return { + data: await this.providers.upsert( + principal.principalId, + principal.workspaceId, + capability, + body, + ), + }; + } + + @Delete(':capability') + remove( + @Req() request: Request, + @Param('capability') capability: CapabilityName, + ) { + return this.providers.remove(requester(request).principalId, capability); + } +} + +function requester(request: Request) { + const principal = (request as any).principal as ApiKeyPrincipal | undefined; + const principalId = resolveCallerId(request); + if (!principalId) throw new Error('Authenticated principal required'); + return { principalId, workspaceId: principal?.workspaceId ?? null }; +} diff --git a/apps/commons-api/src/provider/capability-provider.module.ts b/apps/commons-api/src/provider/capability-provider.module.ts new file mode 100644 index 00000000..8ba18737 --- /dev/null +++ b/apps/commons-api/src/provider/capability-provider.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { CapabilityProviderController } from './capability-provider.controller'; +import { CapabilityProviderService } from './capability-provider.service'; + +@Module({ + controllers: [CapabilityProviderController], + providers: [CapabilityProviderService], + exports: [CapabilityProviderService], +}) +export class CapabilityProviderModule {} diff --git a/apps/commons-api/src/provider/capability-provider.service.spec.ts b/apps/commons-api/src/provider/capability-provider.service.spec.ts new file mode 100644 index 00000000..0cd5df44 --- /dev/null +++ b/apps/commons-api/src/provider/capability-provider.service.spec.ts @@ -0,0 +1,38 @@ +import { CapabilityProviderService } from './capability-provider.service'; + +describe('CapabilityProviderService validation', () => { + const service = new CapabilityProviderService({} as any, {} as any); + + it.each([ + 'http://search.example.com', + 'https://localhost/search', + 'https://127.0.0.1/search', + 'https://10.0.0.4/search', + 'https://192.168.1.8/search', + ])('rejects unsafe custom endpoints: %s', async (endpointUrl) => { + await expect( + service.upsert('user-1', null, 'web_search', { + provider: 'custom', + endpointUrl, + }), + ).rejects.toThrow(); + }); + + it('keeps credentials out of provider settings', async () => { + await expect( + service.upsert('user-1', null, 'web_search', { + provider: 'custom', + endpointUrl: 'https://search.example.com', + settings: { apiKey: 'should-not-be-here' }, + }), + ).rejects.toThrow('Store apiKey in credentials'); + }); + + it('rejects unknown capabilities at the API boundary', async () => { + await expect( + service.upsert('user-1', null, 'email' as any, { + provider: 'custom', + }), + ).rejects.toThrow('Unsupported capability'); + }); +}); diff --git a/apps/commons-api/src/provider/capability-provider.service.ts b/apps/commons-api/src/provider/capability-provider.service.ts new file mode 100644 index 00000000..d50ccddc --- /dev/null +++ b/apps/commons-api/src/provider/capability-provider.service.ts @@ -0,0 +1,292 @@ +import { + BadRequestException, + Injectable, + NotFoundException, +} from '@nestjs/common'; +import { and, eq, sql } from 'drizzle-orm'; +import { DatabaseService } from '~/modules/database/database.service'; +import { EncryptionService } from '~/modules/encryption/encryption.service'; +import { capabilityProvider as capabilityProviderTable } from '#/models/schema'; + +export type CapabilityName = 'web_search' | 'computer' | 'wallet'; + +export type CapabilityProviderInput = { + provider: string; + displayName?: string; + endpointUrl?: string; + settings?: Record; + credentials?: Record; + status?: 'active' | 'disabled'; +}; + +export type ResolvedCapabilityProvider = { + provider: string; + endpointUrl?: string; + settings: Record; + credentials: Record; + source: 'account' | 'platform'; +}; + +const CATALOG = { + web_search: [ + { id: 'platform', name: 'Commons Search', credentialFields: [] }, + { id: 'brave', name: 'Brave Search', credentialFields: ['apiKey'] }, + { id: 'tavily', name: 'Tavily', credentialFields: ['apiKey'] }, + { + id: 'searxng', + name: 'SearXNG', + credentialFields: ['apiKey'], + endpoint: true, + }, + { + id: 'custom', + name: 'Custom search API', + credentialFields: ['apiKey'], + endpoint: true, + }, + ], + computer: [ + { id: 'commonos', name: 'CommonOS', credentialFields: [] }, + { + id: 'custom', + name: 'Custom computer adapter', + credentialFields: ['apiKey'], + endpoint: true, + }, + ], + wallet: [ + { id: 'commons_mpc', name: 'Commons managed wallet', credentialFields: [] }, + { id: 'external', name: 'Owner-connected wallet', credentialFields: [] }, + { + id: 'custom', + name: 'Custom wallet adapter', + credentialFields: ['apiKey'], + endpoint: true, + }, + ], +} as const; + +@Injectable() +export class CapabilityProviderService { + constructor( + private readonly db: DatabaseService, + private readonly encryption: EncryptionService, + ) {} + + catalog() { + return CATALOG; + } + + async list(ownerId: string) { + const rows = await this.db + .select() + .from(capabilityProviderTable) + .where( + sql`lower(${capabilityProviderTable.ownerId}) = lower(${ownerId})`, + ) + .orderBy(capabilityProviderTable.capability); + return { + catalog: CATALOG, + configurations: rows.map((row) => this.publicConfiguration(row)), + }; + } + + async upsert( + ownerId: string, + workspaceId: string | null | undefined, + capability: CapabilityName, + input: CapabilityProviderInput, + ) { + this.assertCapability(capability); + const definition = this.definition(capability, input.provider); + const endpointUrl = input.endpointUrl?.trim() || null; + if ('endpoint' in definition && definition.endpoint && !endpointUrl) { + throw new BadRequestException( + `${definition.name} requires an endpoint URL`, + ); + } + if (endpointUrl) validateEndpoint(endpointUrl); + const settings = sanitizeSettings(input.settings ?? {}); + const existing = await this.find(ownerId, capability); + let encrypted = existing + ? { + encryptedCredentials: existing.encryptedCredentials, + credentialsIv: existing.credentialsIv, + credentialsTag: existing.credentialsTag, + } + : { + encryptedCredentials: null as string | null, + credentialsIv: null as string | null, + credentialsTag: null as string | null, + }; + if (input.credentials && Object.keys(input.credentials).length) { + const credentials = cleanCredentials(input.credentials); + const sealed = this.encryption.encrypt(JSON.stringify(credentials)); + encrypted = { + encryptedCredentials: sealed.encryptedValue, + credentialsIv: sealed.iv, + credentialsTag: sealed.tag, + }; + } + const values = { + ownerId, + workspaceId: workspaceId ?? null, + capability, + provider: input.provider, + displayName: input.displayName?.trim() || definition.name, + endpointUrl, + settings, + ...encrypted, + status: input.status ?? 'active', + updatedAt: new Date(), + }; + const [saved] = await this.db + .insert(capabilityProviderTable) + .values(values) + .onConflictDoUpdate({ + target: [ + capabilityProviderTable.ownerId, + capabilityProviderTable.capability, + ], + set: values, + }) + .returning(); + return this.publicConfiguration(saved); + } + + async remove(ownerId: string, capability: CapabilityName) { + this.assertCapability(capability); + const removed = await this.db + .delete(capabilityProviderTable) + .where( + and( + sql`lower(${capabilityProviderTable.ownerId}) = lower(${ownerId})`, + eq(capabilityProviderTable.capability, capability), + ), + ) + .returning({ id: capabilityProviderTable.id }); + if (!removed.length) + throw new NotFoundException('Provider configuration not found'); + return { deleted: true }; + } + + async resolve( + ownerId: string, + capability: CapabilityName, + ): Promise { + this.assertCapability(capability); + const row = await this.find(ownerId, capability); + if (!row || row.status !== 'active' || row.provider === 'platform') + return null; + let credentials: Record = {}; + if (row.encryptedCredentials && row.credentialsIv && row.credentialsTag) { + const plaintext = this.encryption.decrypt( + row.encryptedCredentials, + row.credentialsIv, + row.credentialsTag, + ); + credentials = JSON.parse(plaintext); + } + return { + provider: row.provider, + endpointUrl: row.endpointUrl ?? undefined, + settings: (row.settings ?? {}) as Record, + credentials, + source: 'account', + }; + } + + private definition(capability: CapabilityName, provider: string) { + const definition = CATALOG[capability].find((item) => item.id === provider); + if (!definition) { + throw new BadRequestException( + `Unsupported ${capability} provider "${provider}"`, + ); + } + return definition; + } + + private assertCapability(capability: CapabilityName) { + if (!Object.prototype.hasOwnProperty.call(CATALOG, capability)) { + throw new BadRequestException(`Unsupported capability "${capability}"`); + } + } + + private find(ownerId: string, capability: CapabilityName) { + return this.db.query.capabilityProvider.findFirst({ + where: (table) => + and( + sql`lower(${table.ownerId}) = lower(${ownerId})`, + eq(table.capability, capability), + ), + }); + } + + private publicConfiguration( + row: typeof capabilityProviderTable.$inferSelect, + ) { + return { + id: row.id, + capability: row.capability, + provider: row.provider, + displayName: row.displayName, + endpointUrl: row.endpointUrl, + settings: row.settings ?? {}, + status: row.status, + hasCredentials: Boolean(row.encryptedCredentials), + updatedAt: row.updatedAt, + }; + } +} + +function cleanCredentials(credentials: Record) { + return Object.fromEntries( + Object.entries(credentials) + .map(([key, value]) => [key.trim(), value.trim()]) + .filter(([key, value]) => Boolean(key && value)), + ); +} + +function sanitizeSettings(settings: Record) { + for (const key of Object.keys(settings)) { + if (/(secret|token|password|api.?key|private.?key|credential)/i.test(key)) { + throw new BadRequestException( + `Store ${key} in credentials, not settings`, + ); + } + } + return settings; +} + +function validateEndpoint(value: string) { + let endpoint: URL; + try { + endpoint = new URL(value); + } catch { + throw new BadRequestException('Provider endpoint must be a valid URL'); + } + if (endpoint.protocol !== 'https:') { + throw new BadRequestException( + 'User-provided provider endpoints must use HTTPS', + ); + } + if (!endpoint.hostname || isPrivateHostname(endpoint.hostname)) { + throw new BadRequestException( + 'Provider endpoint must use a public hostname', + ); + } +} + +function isPrivateHostname(hostname: string) { + const normalized = hostname.toLowerCase(); + return ( + normalized === 'localhost' || + normalized.endsWith('.local') || + normalized === '::1' || + /^127\./.test(normalized) || + /^10\./.test(normalized) || + /^192\.168\./.test(normalized) || + /^169\.254\./.test(normalized) || + /^172\.(1[6-9]|2\d|3[01])\./.test(normalized) + ); +} diff --git a/apps/commons-api/src/provider/index.ts b/apps/commons-api/src/provider/index.ts new file mode 100644 index 00000000..ad42346c --- /dev/null +++ b/apps/commons-api/src/provider/index.ts @@ -0,0 +1,2 @@ +export * from './capability-provider.module'; +export * from './capability-provider.service'; diff --git a/apps/commons-api/src/skill/skill.controller.ts b/apps/commons-api/src/skill/skill.controller.ts index 4d5eeebe..84628da7 100644 --- a/apps/commons-api/src/skill/skill.controller.ts +++ b/apps/commons-api/src/skill/skill.controller.ts @@ -7,9 +7,16 @@ import { Param, Body, Query, + Req, HttpCode, HttpStatus, + UploadedFile, + UseInterceptors, } from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { memoryStorage } from 'multer'; +import type { Request } from 'express'; +import { resolveCallerId, type ApiKeyPrincipal } from '~/modules/auth'; import { SkillService, CreateSkillDto } from './skill.service'; @Controller({ version: '1', path: 'skills' }) @@ -18,6 +25,7 @@ export class SkillController { @Get() async list( + @Req() req: Request, @Query('ownerId') ownerId?: string, @Query('ownerType') ownerType?: string, @Query('isPublic') isPublic?: string, @@ -27,37 +35,102 @@ export class SkillController { if (ownerType) filter.ownerType = ownerType; if (isPublic !== undefined) filter.isPublic = isPublic === 'true'; - const skills = await this.skillService.list(Object.keys(filter).length ? filter : undefined); + const principal = principalFrom(req); + const skills = await this.skillService.list( + Object.keys(filter).length ? filter : undefined, + principal.principalId, + ); return { data: skills }; } @Get('index') - async getIndex(@Query('ownerId') ownerId?: string) { - const index = await this.skillService.getIndex(ownerId); + async getIndex(@Req() req: Request, @Query('ownerId') ownerId?: string) { + const index = await this.skillService.getIndex( + ownerId, + principalFrom(req), + ); return { data: index }; } + @Get('agents/:agentId') + async listForAgent(@Req() req: Request, @Param('agentId') agentId: string) { + return { + data: await this.skillService.listForAgent(agentId, principalFrom(req)), + }; + } + + @Put(':id/agents/:agentId') + async assignToAgent( + @Req() req: Request, + @Param('id') id: string, + @Param('agentId') agentId: string, + @Body() body: { isEnabled?: boolean }, + ) { + return { + data: await this.skillService.assignToAgent( + id, + agentId, + body.isEnabled !== false, + principalFrom(req), + ), + }; + } + @Get(':id') - async get(@Param('id') id: string) { - const skill = await this.skillService.get(id); + async get(@Req() req: Request, @Param('id') id: string) { + const skill = await this.skillService.get(id, principalFrom(req)); return { data: skill }; } @Post() - async create(@Body() dto: CreateSkillDto) { - const skill = await this.skillService.create(dto); + async create(@Req() req: Request, @Body() dto: CreateSkillDto) { + const skill = await this.skillService.create(dto, principalFrom(req)); return { data: skill }; } + @Post('import') + @UseInterceptors( + FileInterceptor('file', { + storage: memoryStorage(), + limits: { fileSize: 10 * 1024 * 1024, files: 1 }, + }), + ) + async importSkill( + @Req() req: Request, + @UploadedFile() file: Express.Multer.File | undefined, + ) { + return { + data: await this.skillService.importSkillFile(file, principalFrom(req)), + }; + } + @Put(':id') - async update(@Param('id') id: string, @Body() updates: Partial) { - const skill = await this.skillService.update(id, updates); + async update( + @Req() req: Request, + @Param('id') id: string, + @Body() updates: Partial, + ) { + const skill = await this.skillService.update( + id, + updates, + principalFrom(req), + ); return { data: skill }; } @Delete(':id') @HttpCode(HttpStatus.OK) - async delete(@Param('id') id: string) { - return this.skillService.delete(id); + async delete(@Req() req: Request, @Param('id') id: string) { + return this.skillService.delete(id, principalFrom(req)); } } + +function principalFrom(req: Request) { + const principal = (req as any).principal as ApiKeyPrincipal | undefined; + const principalId = resolveCallerId(req); + if (!principalId) throw new Error('Authenticated principal required'); + return { + principalId, + workspaceId: principal?.workspaceId, + }; +} diff --git a/apps/commons-api/src/skill/skill.service.spec.ts b/apps/commons-api/src/skill/skill.service.spec.ts index 53116c57..f8e3869f 100644 --- a/apps/commons-api/src/skill/skill.service.spec.ts +++ b/apps/commons-api/src/skill/skill.service.spec.ts @@ -1,4 +1,5 @@ import { SkillService } from './skill.service'; +import JSZip from 'jszip'; describe('SkillService prompt index', () => { it('tells native agents to load matching skills before artifact execution', async () => { @@ -58,3 +59,66 @@ describe('SkillService prompt index', () => { expect(prompt).toContain('MUST still call invoke_skill'); }); }); + +describe('SkillService imports', () => { + it('imports a portable SKILL.md with triggers, tools, and tags', async () => { + const database = { + query: { skill: { findFirst: jest.fn().mockResolvedValue(null) } }, + } as any; + const service = new SkillService(database); + const create = jest.spyOn(service, 'create').mockResolvedValue({ + skillId: 'skill-1', + name: 'Weekly Report', + } as any); + const markdown = `--- +name: weekly-report +title: Weekly Report +description: Prepare a concise weekly progress report. +tools: [readUploadedFile, web_search] +triggers: + - weekly update + - progress report +tags: [writing, reporting] +--- +## Workflow + +Gather evidence, draft the report, and validate every claim.`; + + await service.importSkillFile( + { + originalname: 'SKILL.md', + mimetype: 'text/markdown', + buffer: Buffer.from(markdown), + } as Express.Multer.File, + { principalId: 'user-1' }, + ); + + expect(create).toHaveBeenCalledWith( + expect.objectContaining({ + slug: 'weekly-report', + name: 'Weekly Report', + tools: ['readUploadedFile', 'web_search'], + triggers: ['weekly update', 'progress report'], + tags: ['writing', 'reporting'], + source: 'import', + }), + { principalId: 'user-1' }, + ); + }); + + it('rejects an archive without SKILL.md', async () => { + const archive = new JSZip(); + archive.file('README.md', '# Not a skill'); + const service = new SkillService({} as any); + await expect( + service.importSkillFile( + { + originalname: 'broken.skill', + mimetype: 'application/zip', + buffer: await archive.generateAsync({ type: 'nodebuffer' }), + } as Express.Multer.File, + { principalId: 'user-1' }, + ), + ).rejects.toThrow('must contain a SKILL.md'); + }); +}); diff --git a/apps/commons-api/src/skill/skill.service.ts b/apps/commons-api/src/skill/skill.service.ts index 25998f66..2c6f90a3 100644 --- a/apps/commons-api/src/skill/skill.service.ts +++ b/apps/commons-api/src/skill/skill.service.ts @@ -1,12 +1,19 @@ import { + BadRequestException, + ForbiddenException, Injectable, Logger, NotFoundException, OnModuleInit, } from '@nestjs/common'; +import JSZip from 'jszip'; import { DatabaseService } from '../modules/database/database.service'; -import { skill as skillTable } from '../../models/schema'; -import { eq, and, or, sql } from 'drizzle-orm'; +import { + agent as agentTable, + agentSkill as agentSkillTable, + skill as skillTable, +} from '../../models/schema'; +import { eq, and, or, sql, inArray } from 'drizzle-orm'; import { readdir, readFile } from 'node:fs/promises'; import path from 'node:path'; @@ -36,6 +43,20 @@ export interface SkillIndex { triggers: string[]; } +export interface SkillAgentAssignment { + assignmentId: string; + agentId: string; + agentName: string; + agentAvatar?: string | null; + isDefault: boolean; + isEnabled: boolean; +} + +export interface SkillRequester { + principalId: string; + workspaceId?: string | null; +} + @Injectable() export class SkillService implements OnModuleInit { private readonly logger = new Logger(SkillService.name); @@ -52,11 +73,14 @@ export class SkillService implements OnModuleInit { ); } - async list(filter?: { - ownerId?: string; - ownerType?: string; - isPublic?: boolean; - }) { + async list( + filter?: { + ownerId?: string; + ownerType?: string; + isPublic?: boolean; + }, + viewerOwnerId?: string, + ) { const conditions: any[] = []; if (filter?.ownerId) @@ -66,14 +90,38 @@ export class SkillService implements OnModuleInit { if (filter?.isPublic !== undefined) conditions.push(eq(skillTable.isPublic, filter.isPublic)); - return this.db + if (viewerOwnerId) { + const viewerAgents = await this.db.query.agent.findMany({ + where: (table) => + or( + sql`lower(${table.ownerUserId}) = lower(${viewerOwnerId})`, + sql`lower(${table.owner}) = lower(${viewerOwnerId})`, + ), + columns: { agentId: true }, + }); + const privateOwners = [ + sql`lower(${skillTable.ownerId}) = lower(${viewerOwnerId})`, + ...(viewerAgents.length + ? [ + inArray( + skillTable.ownerId, + viewerAgents.map((agent) => agent.agentId), + ), + ] + : []), + ]; + conditions.push(or(eq(skillTable.isPublic, true), ...privateOwners)); + } + + const skills = await this.db .select() .from(skillTable) .where(conditions.length ? and(...conditions) : undefined) .orderBy(skillTable.name); + return this.withAgentAssignments(skills, viewerOwnerId); } - async get(skillIdOrSlug: string) { + async get(skillIdOrSlug: string, requester?: SkillRequester) { const rows = await this.db .select() .from(skillTable) @@ -88,10 +136,29 @@ export class SkillService implements OnModuleInit { if (!rows.length) { throw new NotFoundException(`Skill "${skillIdOrSlug}" not found`); } - return rows[0]; + const found = rows[0]; + await this.assertCanViewSkill(found, requester); + return found; } - async create(dto: CreateSkillDto) { + async create(dto: CreateSkillDto, requester?: SkillRequester) { + let ownerType = dto.ownerType ?? 'user'; + let ownerId = dto.ownerId ?? requester?.principalId ?? null; + if (requester) { + if (ownerType === 'platform') { + throw new ForbiddenException( + 'Platform skills are managed by the platform', + ); + } + if (ownerType === 'agent') { + if (!ownerId) + throw new ForbiddenException('An agent owner is required'); + await this.requireManageableAgent(ownerId, requester); + } else { + ownerType = 'user'; + ownerId = requester.principalId; + } + } const rows = await this.db .insert(skillTable) .values({ @@ -101,8 +168,8 @@ export class SkillService implements OnModuleInit { instructions: dto.instructions, tools: dto.tools ?? [], triggers: dto.triggers ?? [], - ownerId: dto.ownerId ?? null, - ownerType: dto.ownerType ?? 'user', + ownerId, + ownerType, isPublic: dto.isPublic ?? false, tags: dto.tags ?? [], icon: dto.icon ?? null, @@ -111,11 +178,99 @@ export class SkillService implements OnModuleInit { }) .returning(); - return rows[0]; + const created = rows[0]; + if (created && ownerType === 'agent' && ownerId) { + await this.setAgentAssignment({ + agentId: ownerId, + skillId: created.skillId, + isEnabled: true, + assignedBy: requester?.principalId ?? ownerId, + }); + } + return created; } - async update(skillIdOrSlug: string, updates: Partial) { + async importSkillFile( + file: Express.Multer.File | undefined, + requester: SkillRequester, + ) { + if (!file?.buffer?.length) { + throw new BadRequestException('Choose a .md, .zip, or .skill file'); + } + let markdown = ''; + const fileName = file.originalname.toLowerCase(); + if (fileName.endsWith('.md') || file.mimetype === 'text/markdown') { + markdown = file.buffer.toString('utf8'); + } else if (fileName.endsWith('.zip') || fileName.endsWith('.skill')) { + const archive = await JSZip.loadAsync(file.buffer).catch(() => null); + if (!archive) + throw new BadRequestException('The skill archive is invalid'); + const entries = Object.values(archive.files).filter( + (entry) => !entry.dir, + ); + if (entries.length > 500) { + throw new BadRequestException( + 'The skill archive contains too many files', + ); + } + if ( + entries.some((entry) => + entry.name.split('/').some((segment) => segment === '..'), + ) + ) { + throw new BadRequestException( + 'The skill archive contains an unsafe path', + ); + } + const skillFile = entries + .filter((entry) => /(^|\/)skill\.md$/i.test(entry.name)) + .sort((left, right) => left.name.length - right.name.length)[0]; + if (!skillFile) { + throw new BadRequestException( + 'The archive must contain a SKILL.md file', + ); + } + markdown = await skillFile.async('string'); + } else { + throw new BadRequestException( + 'Skills must be .md, .zip, or .skill files', + ); + } + if (Buffer.byteLength(markdown, 'utf8') > 1024 * 1024) { + throw new BadRequestException('SKILL.md must be smaller than 1 MB'); + } + const parsed = parseSkillMarkdown(markdown); + const collision = await this.db.query.skill.findFirst({ + where: (table) => eq(table.slug, parsed.slug), + columns: { skillId: true }, + }); + const slug = collision + ? `${parsed.slug}-import-${Date.now().toString(36)}` + : parsed.slug; + return this.create( + { + slug, + name: parsed.title ?? titleFromSlug(parsed.slug), + description: parsed.description, + instructions: parsed.instructions, + tools: parsed.tools, + triggers: parsed.triggers, + tags: parsed.tags, + isPublic: false, + ownerType: 'user', + source: 'import', + }, + requester, + ); + } + + async update( + skillIdOrSlug: string, + updates: Partial, + requester?: SkillRequester, + ) { const existing = await this.get(skillIdOrSlug); + await this.assertCanManageSkill(existing, requester); const rows = await this.db .update(skillTable) @@ -140,8 +295,9 @@ export class SkillService implements OnModuleInit { return rows[0]; } - async delete(skillIdOrSlug: string) { + async delete(skillIdOrSlug: string, requester?: SkillRequester) { const existing = await this.get(skillIdOrSlug); + await this.assertCanManageSkill(existing, requester); await this.db .delete(skillTable) .where(eq(skillTable.skillId, existing.skillId)); @@ -161,17 +317,12 @@ export class SkillService implements OnModuleInit { * Returns the compact index (no full instructions) for progressive disclosure. * Used at session start to give the agent a lightweight menu of available skills. */ - async getIndex(ownerId?: string): Promise { - const conditions: any[] = [eq(skillTable.isActive, true)]; - - if (ownerId) { - conditions.push( - or(eq(skillTable.isPublic, true), eq(skillTable.ownerId, ownerId)), - ); - } else { - conditions.push(eq(skillTable.isPublic, true)); - } - + async getIndex( + ownerId?: string, + requester?: SkillRequester, + ): Promise { + if (!ownerId) return []; + if (requester) await this.requireManageableAgent(ownerId, requester); return this.db .select({ skillId: skillTable.skillId, @@ -182,9 +333,288 @@ export class SkillService implements OnModuleInit { icon: skillTable.icon, triggers: skillTable.triggers, }) + .from(agentSkillTable) + .innerJoin(skillTable, eq(agentSkillTable.skillId, skillTable.skillId)) + .where( + and( + eq(agentSkillTable.agentId, ownerId), + eq(agentSkillTable.isEnabled, true), + eq(skillTable.isActive, true), + ), + ) + .orderBy(skillTable.name); + } + + async getForAgent(skillIdOrSlug: string, agentId: string) { + const rows = await this.db + .select({ skill: skillTable }) + .from(agentSkillTable) + .innerJoin(skillTable, eq(agentSkillTable.skillId, skillTable.skillId)) + .where( + and( + eq(agentSkillTable.agentId, agentId), + eq(agentSkillTable.isEnabled, true), + eq(skillTable.isActive, true), + or( + eq(skillTable.skillId, skillIdOrSlug), + eq(skillTable.slug, skillIdOrSlug), + ), + ), + ) + .limit(1); + if (!rows.length) { + throw new NotFoundException( + `Skill "${skillIdOrSlug}" is not available to this agent`, + ); + } + return rows[0].skill; + } + + async listForAgent(agentId: string, requester?: SkillRequester) { + const target = await this.requireManageableAgent(agentId, requester); + const ownerUserId = target.ownerUserId ?? target.owner; + const siblingAgents = ownerUserId + ? await this.db.query.agent.findMany({ + where: (table) => + or( + eq(table.ownerUserId, ownerUserId), + eq(table.owner, ownerUserId), + ), + columns: { agentId: true }, + }) + : []; + const visibleOwnerIds = [ + ownerUserId, + ...siblingAgents.map((agent) => agent.agentId), + ].filter(Boolean) as string[]; + const skills = await this.db + .select() .from(skillTable) - .where(and(...conditions)) + .where( + and( + eq(skillTable.isActive, true), + or( + eq(skillTable.isPublic, true), + visibleOwnerIds.length + ? inArray(skillTable.ownerId, visibleOwnerIds) + : undefined, + ), + ), + ) .orderBy(skillTable.name); + const assignments = await this.db.query.agentSkill.findMany({ + where: (table) => eq(table.agentId, agentId), + }); + const assignmentBySkill = new Map( + assignments.map((assignment) => [assignment.skillId, assignment]), + ); + return skills.map((skill) => { + const assignment = assignmentBySkill.get(skill.skillId); + return { + ...skill, + assignmentId: assignment?.id ?? null, + assigned: Boolean(assignment?.isEnabled), + }; + }); + } + + async assignToAgent( + skillIdOrSlug: string, + agentId: string, + isEnabled: boolean, + requester?: SkillRequester, + ) { + const target = await this.requireManageableAgent(agentId, requester); + const skill = await this.get(skillIdOrSlug); + await this.assertSkillCanBeAssigned(skill, target, requester); + return this.setAgentAssignment({ + agentId, + skillId: skill.skillId, + isEnabled, + assignedBy: requester?.principalId ?? agentId, + }); + } + + async ensurePlatformSkillsForCopilot(agentId: string, assignedBy?: string) { + const platformSkills = await this.db + .select({ skillId: skillTable.skillId }) + .from(skillTable) + .where( + and( + eq(skillTable.ownerType, 'platform'), + eq(skillTable.isActive, true), + ), + ); + if (!platformSkills.length) return; + await this.db + .insert(agentSkillTable) + .values( + platformSkills.map((skill) => ({ + agentId, + skillId: skill.skillId, + isEnabled: true, + assignedBy: assignedBy ?? agentId, + })), + ) + .onConflictDoNothing(); + } + + private async setAgentAssignment(input: { + agentId: string; + skillId: string; + isEnabled: boolean; + assignedBy: string; + }) { + const [assignment] = await this.db + .insert(agentSkillTable) + .values({ + agentId: input.agentId, + skillId: input.skillId, + isEnabled: input.isEnabled, + assignedBy: input.assignedBy, + }) + .onConflictDoUpdate({ + target: [agentSkillTable.agentId, agentSkillTable.skillId], + set: { + isEnabled: input.isEnabled, + assignedBy: input.assignedBy, + updatedAt: new Date(), + }, + }) + .returning(); + return assignment; + } + + private async requireManageableAgent( + agentId: string, + requester?: SkillRequester, + ) { + const target = await this.db.query.agent.findFirst({ + where: (table) => eq(table.agentId, agentId), + }); + if (!target) throw new NotFoundException('Agent not found'); + if (!requester) return target; + const ownsAgent = [target.ownerUserId, target.owner] + .filter(Boolean) + .some( + (owner) => owner!.toLowerCase() === requester.principalId.toLowerCase(), + ); + const sharesWorkspace = Boolean( + requester.workspaceId && + target.workspaceId && + requester.workspaceId.toLowerCase() === + target.workspaceId.toLowerCase(), + ); + if (!ownsAgent && !sharesWorkspace) { + throw new ForbiddenException('You cannot manage skills for this agent'); + } + return target; + } + + private async assertCanViewSkill( + skill: typeof skillTable.$inferSelect, + requester?: SkillRequester, + ) { + if (!requester || skill.isPublic) return; + if (sameIdentity(skill.ownerId, requester.principalId)) return; + if (skill.ownerType === 'agent' && skill.ownerId) { + await this.requireManageableAgent(skill.ownerId, requester); + return; + } + throw new ForbiddenException('You cannot access this skill'); + } + + private async assertCanManageSkill( + skill: typeof skillTable.$inferSelect, + requester?: SkillRequester, + ) { + if (!requester) return; + if (skill.ownerType === 'platform') { + throw new ForbiddenException( + 'Platform skills are managed by the platform', + ); + } + if (skill.ownerType === 'agent' && skill.ownerId) { + await this.requireManageableAgent(skill.ownerId, requester); + return; + } + if (sameIdentity(skill.ownerId, requester.principalId)) return; + throw new ForbiddenException('You cannot manage this skill'); + } + + private async assertSkillCanBeAssigned( + skill: typeof skillTable.$inferSelect, + target: typeof agentTable.$inferSelect, + requester?: SkillRequester, + ) { + if (!requester || skill.isPublic) return; + if (sameIdentity(skill.ownerId, requester.principalId)) return; + if (sameIdentity(skill.ownerId, target.agentId)) return; + if (skill.ownerType === 'agent' && skill.ownerId) { + const source = await this.db.query.agent.findFirst({ + where: (table) => eq(table.agentId, skill.ownerId!), + }); + const targetOwner = target.ownerUserId ?? target.owner; + const sourceOwner = source?.ownerUserId ?? source?.owner; + const sameOwner = sameIdentity(targetOwner, sourceOwner); + const sameWorkspace = Boolean( + target.workspaceId && + source?.workspaceId && + sameIdentity(target.workspaceId, source.workspaceId), + ); + if (sameOwner || sameWorkspace) return; + } + throw new ForbiddenException('This skill is not available to this agent'); + } + + private async withAgentAssignments( + skills: Array, + viewerOwnerId?: string, + ) { + if (!skills.length || !viewerOwnerId) { + return skills.map((skill) => ({ ...skill, assignedAgents: [] })); + } + const rows = await this.db + .select({ + assignmentId: agentSkillTable.id, + skillId: agentSkillTable.skillId, + isEnabled: agentSkillTable.isEnabled, + agentId: agentTable.agentId, + agentName: agentTable.name, + agentAvatar: agentTable.avatar, + isDefault: agentTable.isDefault, + }) + .from(agentSkillTable) + .innerJoin(agentTable, eq(agentSkillTable.agentId, agentTable.agentId)) + .where( + and( + inArray( + agentSkillTable.skillId, + skills.map((skill) => skill.skillId), + ), + or( + sql`lower(${agentTable.ownerUserId}) = lower(${viewerOwnerId})`, + sql`lower(${agentTable.owner}) = lower(${viewerOwnerId})`, + ), + ), + ); + const bySkill = new Map(); + for (const row of rows) { + const assignments = bySkill.get(row.skillId) ?? []; + assignments.push({ + assignmentId: row.assignmentId, + agentId: row.agentId, + agentName: row.agentName, + agentAvatar: row.agentAvatar, + isDefault: row.isDefault, + isEnabled: row.isEnabled, + }); + bySkill.set(row.skillId, assignments); + } + return skills.map((skill) => ({ + ...skill, + assignedAgents: bySkill.get(skill.skillId) ?? [], + })); } async buildPromptIndex(ownerId?: string, requestText = '') { @@ -298,6 +728,17 @@ export class SkillService implements OnModuleInit { synced += 1; } this.logger.log(`Synced ${synced} bundled platform skills`); + const copilots = await this.db.query.agent.findMany({ + where: (table) => + and(eq(table.isDefault, true), eq(table.isSystemManaged, true)), + columns: { agentId: true, ownerUserId: true, owner: true }, + }); + for (const copilot of copilots) { + await this.ensurePlatformSkillsForCopilot( + copilot.agentId, + copilot.ownerUserId ?? copilot.owner ?? copilot.agentId, + ); + } } } @@ -379,6 +820,25 @@ const BUNDLED_SKILL_CONFIG: Record< icon: 'monitor', version: '1.0.0', }, + 'build-commons-ui-plugin': { + name: 'Build Commons UI Plugins', + tools: [ + 'createCodeProject', + 'writeCodeProjectFiles', + 'publishCodeProject', + 'testCodeProject', + 'registerUiPlugin', + ], + triggers: [ + 'custom UI', + 'custom page', + 'floating widget', + 'Commons app plugin', + ], + tags: ['ui', 'plugin', 'widget', 'app'], + icon: 'panels-top-left', + version: '1.0.0', + }, }; async function findBundledSkillDirectory() { @@ -413,7 +873,44 @@ function parseSkillMarkdown(source: string) { } return { slug, + title: value('title'), description, instructions: match[2].trim(), + tools: frontmatterList(frontmatter, 'tools'), + triggers: frontmatterList(frontmatter, 'triggers'), + tags: frontmatterList(frontmatter, 'tags'), }; } + +function frontmatterList(frontmatter: string, field: string) { + const inline = frontmatter.match( + new RegExp(`^${field}:\\s*\\[([^\\]]*)\\]`, 'm'), + )?.[1]; + if (inline !== undefined) { + return inline + .split(',') + .map((value) => value.trim().replace(/^['"]|['"]$/g, '')) + .filter(Boolean); + } + const block = frontmatter.match( + new RegExp(`^${field}:\\s*\\r?\\n((?:\\s+-\\s+.+\\r?\\n?)*)`, 'm'), + )?.[1]; + return (block?.match(/^\s+-\s+(.+)$/gm) ?? []).map((line) => + line + .replace(/^\s+-\s+/, '') + .replace(/^['"]|['"]$/g, '') + .trim(), + ); +} + +function titleFromSlug(slug: string) { + return slug + .split(/[-_]+/) + .filter(Boolean) + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(' '); +} + +function sameIdentity(left?: string | null, right?: string | null) { + return Boolean(left && right && left.toLowerCase() === right.toLowerCase()); +} diff --git a/apps/commons-api/src/tool/tool.module.ts b/apps/commons-api/src/tool/tool.module.ts index 174fd605..c9f195c0 100644 --- a/apps/commons-api/src/tool/tool.module.ts +++ b/apps/commons-api/src/tool/tool.module.ts @@ -25,6 +25,8 @@ import { FilesModule } from '~/files'; import { ComputerModule } from '~/computer'; import { CodeProjectModule } from '~/code-project'; import { UsageModule } from '~/modules/usage'; +import { CapabilityProviderModule } from '~/provider'; +import { UiPluginModule } from '~/ui-plugin'; @Module({ imports: [ @@ -40,6 +42,8 @@ import { UsageModule } from '~/modules/usage'; CodeProjectModule, UsageModule, OAuthModule, + CapabilityProviderModule, + UiPluginModule, ], controllers: [ ToolController, diff --git a/apps/commons-api/src/tool/tools/common-tool.service.ts b/apps/commons-api/src/tool/tools/common-tool.service.ts index 34013c90..4752041d 100644 --- a/apps/commons-api/src/tool/tools/common-tool.service.ts +++ b/apps/commons-api/src/tool/tools/common-tool.service.ts @@ -35,8 +35,15 @@ import * as schema from '#/models/schema'; import { eq } from 'drizzle-orm'; import { executeWebSearch, + resolveAccountWebSearchConfig, resolveWebSearchConfig, } from './web-search.provider'; +import { CapabilityProviderService } from '~/provider'; +import { + UiPluginService, + type UiPluginPermission, + type UiPluginSurface, +} from '~/ui-plugin'; type ToolExecutionMetadata = { agentId?: string; @@ -652,6 +659,21 @@ export interface CommonTool { projectId: string; }): Promise; + /** + * Register a published code project as a reviewable Commons page or floating + * widget. The plugin remains a draft until its owner explicitly enables it. + */ + registerUiPlugin(props: { + agentId?: string; + codeProjectId: string; + name: string; + slug?: string; + description?: string; + version?: string; + surfaces: UiPluginSurface[]; + permissions?: UiPluginPermission[]; + }): Promise; + /** Test the public project in desktop and mobile Chromium viewports. */ testCodeProject(props: { agentId?: string; @@ -856,6 +878,8 @@ export class CommonToolService { private moduleRef: ModuleRef, private db: DatabaseService, private usage: UsageService, + private capabilityProviders: CapabilityProviderService, + private uiPlugins: UiPluginService, ) {} private async capabilityOwner(agentId: string) { @@ -1103,15 +1127,21 @@ export class CommonToolService { throw new BadRequestException('webSearch requires a non-empty query'); } - const config = resolveWebSearchConfig(); const operationId = metadata?.toolCallId || randomUUID(); const count = Math.max(1, Math.min(props.count ?? 8, 20)); const agentId = this.requireToolAgentId(undefined, metadata); + const owner = await this.capabilityOwner(agentId); + const accountProvider = await this.capabilityProviders.resolve( + owner.principalId, + 'web_search', + ); + const config = accountProvider + ? resolveAccountWebSearchConfig(accountProvider) + : resolveWebSearchConfig(); let reservation: Awaited> = null; if (config.costUsdPerCall > 0) { - const owner = await this.capabilityOwner(agentId); reservation = await this.usage.authorizeCapability({ principalId: owner.principalId, capability: 'web_search', @@ -1455,7 +1485,9 @@ export class CommonToolService { // 2) Construct a gateway URL; you may have PINATA_GATEWAY or custom domain const cid = pinataResult.IpfsHash; - const gatewayUrl = `https://${process.env.GATEWAY_URL ?? 'gateway.pinata.cloud'}/ipfs/${cid}`; + const gatewayUrl = `https://${ + process.env.GATEWAY_URL ?? 'gateway.pinata.cloud' + }/ipfs/${cid}`; // 3) Return IPFS info return { @@ -1824,6 +1856,33 @@ export class CommonToolService { }); } + async registerUiPlugin( + props: { + agentId?: string; + codeProjectId: string; + name: string; + slug?: string; + description?: string; + version?: string; + surfaces: UiPluginSurface[]; + permissions?: UiPluginPermission[]; + }, + metadata?: ToolExecutionMetadata, + ) { + const agentId = this.requireToolAgentId(props.agentId, metadata); + const { surfaces, permissions, ...input } = props; + const plugin = await this.uiPlugins.createForAgent(agentId, { + ...input, + manifest: { schemaVersion: '1', surfaces, permissions }, + }); + return { + ...plugin, + reviewRequired: true, + message: + 'The UI plugin is registered as a draft. Its owner must review and enable it in Studio Apps.', + }; + } + async testCodeProject( props: { agentId?: string; @@ -2121,7 +2180,11 @@ export class CommonToolService { maxTokens, }); - const userContent = `${instruction}\n\nData to process:\n${JSON.stringify(data, null, 2)}\n\nProvide a clear, structured result.`; + const userContent = `${instruction}\n\nData to process:\n${JSON.stringify( + data, + null, + 2, + )}\n\nProvide a clear, structured result.`; const response = await llm.invoke([ new SystemMessage( @@ -2166,7 +2229,7 @@ export class CommonToolService { apiKey?: string; contextId?: string; }, - _metadata?: any, + metadata?: ToolExecutionMetadata, ): Promise<{ text: string; taskId?: string; @@ -2175,7 +2238,11 @@ export class CommonToolService { }> { // ── Local skill lookup ───────────────────────────────────────────────── if (props.skillSlug && !props.url) { - const skill = await this.skillService.get(props.skillSlug); + const agentId = this.requireToolAgentId(undefined, metadata); + const skill = await this.skillService.getForAgent( + props.skillSlug, + agentId, + ); await this.skillService.incrementUsage(props.skillSlug); return { text: `# ${skill.name}\n\n${skill.instructions}`, @@ -2202,7 +2269,9 @@ export class CommonToolService { }; if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`; - const taskId = `skill-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const taskId = `skill-${Date.now()}-${Math.random() + .toString(36) + .slice(2, 8)}`; const rpcBody = { jsonrpc: '2.0', diff --git a/apps/commons-api/src/tool/tools/web-search.provider.spec.ts b/apps/commons-api/src/tool/tools/web-search.provider.spec.ts index 55d6ae0c..e599f154 100644 --- a/apps/commons-api/src/tool/tools/web-search.provider.spec.ts +++ b/apps/commons-api/src/tool/tools/web-search.provider.spec.ts @@ -1,6 +1,7 @@ import { BadRequestException } from '@nestjs/common'; import { executeWebSearch, + resolveAccountWebSearchConfig, resolveWebSearchConfig, } from './web-search.provider'; @@ -104,4 +105,88 @@ describe('web search provider', () => { ), ).rejects.toThrow(/429/); }); + + it('uses a user-supplied Tavily key without platform search configuration', async () => { + const config = resolveAccountWebSearchConfig({ + provider: 'tavily', + credentials: { apiKey: 'tvly-user-key' }, + settings: { searchDepth: 'basic' }, + }); + const fetcher = jest.fn().mockResolvedValue( + new Response( + JSON.stringify({ + results: [ + { + title: 'Current source', + url: 'https://example.com/current', + content: 'Fresh result', + }, + ], + }), + { status: 200 }, + ), + ); + + await executeWebSearch( + config, + { query: 'latest models', count: 5, safeSearch: 'moderate' }, + fetcher, + ); + + expect(fetcher.mock.calls[0][0].toString()).toBe( + 'https://api.tavily.com/search', + ); + expect(fetcher.mock.calls[0][1].method).toBe('POST'); + expect(fetcher.mock.calls[0][1].headers.Authorization).toBe( + 'Bearer tvly-user-key', + ); + expect(JSON.parse(fetcher.mock.calls[0][1].body)).toMatchObject({ + query: 'latest models', + max_results: 5, + }); + }); + + it('maps a custom JSON search response through declarative paths', async () => { + const config = resolveAccountWebSearchConfig({ + provider: 'custom', + endpointUrl: 'https://search.example.com/query', + credentials: { apiKey: 'custom-key' }, + settings: { + resultsPath: 'data.hits', + titlePath: 'name', + urlPath: 'link', + descriptionPath: 'snippet', + }, + }); + const fetcher = jest.fn().mockResolvedValue( + new Response( + JSON.stringify({ + data: { + hits: [ + { + name: 'Mapped title', + link: 'https://example.com/mapped', + snippet: 'Mapped description', + }, + ], + }, + }), + { status: 200 }, + ), + ); + + await expect( + executeWebSearch( + config, + { query: 'portable search', count: 3, safeSearch: 'strict' }, + fetcher, + ), + ).resolves.toEqual([ + { + title: 'Mapped title', + url: 'https://example.com/mapped', + description: 'Mapped description', + }, + ]); + }); }); diff --git a/apps/commons-api/src/tool/tools/web-search.provider.ts b/apps/commons-api/src/tool/tools/web-search.provider.ts index 37ba6cde..b718defb 100644 --- a/apps/commons-api/src/tool/tools/web-search.provider.ts +++ b/apps/commons-api/src/tool/tools/web-search.provider.ts @@ -2,7 +2,7 @@ import { BadRequestException } from '@nestjs/common'; export type WebSearchFreshness = 'day' | 'week' | 'month' | 'year'; export type WebSearchSafeSearch = 'off' | 'moderate' | 'strict'; -export type WebSearchProvider = 'brave' | 'searxng'; +export type WebSearchProvider = 'brave' | 'searxng' | 'tavily' | 'custom'; export type WebSearchInput = { query: string; @@ -24,6 +24,9 @@ export type WebSearchConfig = { braveApiKey?: string; searxngBaseUrl?: string; searxngApiKey?: string; + apiKey?: string; + endpointUrl?: string; + settings?: Record; }; type FetchLike = ( @@ -62,10 +65,10 @@ export function resolveWebSearchConfig( const provider: WebSearchProvider | undefined = requestedProvider ? (requestedProvider as WebSearchProvider) : env.BRAVE_SEARCH_API_KEY - ? 'brave' - : env.SEARXNG_BASE_URL - ? 'searxng' - : undefined; + ? 'brave' + : env.SEARXNG_BASE_URL + ? 'searxng' + : undefined; if (!provider) { throw new BadRequestException(NOT_CONFIGURED_MESSAGE); @@ -106,20 +109,48 @@ export function resolveWebSearchConfig( }; } +export function resolveAccountWebSearchConfig(input: { + provider: string; + endpointUrl?: string; + settings?: Record; + credentials?: Record; +}): WebSearchConfig { + if (!['brave', 'searxng', 'tavily', 'custom'].includes(input.provider)) { + throw new BadRequestException( + `Unsupported web search provider "${input.provider}"`, + ); + } + const provider = input.provider as WebSearchProvider; + const apiKey = input.credentials?.apiKey; + if ((provider === 'brave' || provider === 'tavily') && !apiKey) { + throw new BadRequestException(`${provider} requires an API key`); + } + if ((provider === 'searxng' || provider === 'custom') && !input.endpointUrl) { + throw new BadRequestException(`${provider} requires an endpoint URL`); + } + return { + provider, + costUsdPerCall: 0, + braveApiKey: provider === 'brave' ? apiKey : undefined, + searxngBaseUrl: provider === 'searxng' ? input.endpointUrl : undefined, + searxngApiKey: provider === 'searxng' ? apiKey : undefined, + apiKey, + endpointUrl: input.endpointUrl, + settings: input.settings ?? {}, + }; +} + export async function executeWebSearch( config: WebSearchConfig, input: WebSearchInput, fetcher: FetchLike = fetch, ): Promise { - const request = - config.provider === 'brave' - ? buildBraveRequest(config, input) - : buildSearxngRequest(config, input); + const request = buildSearchRequest(config, input); let response: Response; try { response = await fetcher(request.url, { - headers: request.headers, + ...request.init, signal: AbortSignal.timeout(15_000), }); } catch (error) { @@ -139,7 +170,11 @@ export async function executeWebSearch( const data: any = await response.json(); const rawResults = - config.provider === 'brave' ? data.web?.results : data.results; + config.provider === 'brave' + ? data.web?.results + : config.provider === 'custom' + ? readPath(data, String(config.settings?.resultsPath || 'results')) + : data.results; if (!Array.isArray(rawResults)) return []; return rawResults @@ -151,10 +186,26 @@ export async function executeWebSearch( description: item.description, publishedAt: item.age, } + : config.provider === 'custom' + ? { + title: readPath( + item, + String(config.settings?.titlePath || 'title'), + ), + url: readPath(item, String(config.settings?.urlPath || 'url')), + description: readPath( + item, + String(config.settings?.descriptionPath || 'description'), + ), + publishedAt: readPath( + item, + String(config.settings?.publishedAtPath || 'publishedAt'), + ), + } : { title: item.title, url: item.url, - description: item.content, + description: item.content ?? item.description, publishedAt: item.publishedDate ?? item.published_date, }, ) @@ -168,6 +219,13 @@ export async function executeWebSearch( .slice(0, input.count); } +function buildSearchRequest(config: WebSearchConfig, input: WebSearchInput) { + if (config.provider === 'brave') return buildBraveRequest(config, input); + if (config.provider === 'searxng') return buildSearxngRequest(config, input); + if (config.provider === 'tavily') return buildTavilyRequest(config, input); + return buildCustomRequest(config, input); +} + function buildBraveRequest(config: WebSearchConfig, input: WebSearchInput) { const freshnessMap = { day: 'pd', @@ -184,9 +242,11 @@ function buildBraveRequest(config: WebSearchConfig, input: WebSearchInput) { } return { url, - headers: { - Accept: 'application/json', - 'X-Subscription-Token': config.braveApiKey!, + init: { + headers: { + Accept: 'application/json', + 'X-Subscription-Token': config.braveApiKey!, + }, }, }; } @@ -208,11 +268,74 @@ function buildSearxngRequest(config: WebSearchConfig, input: WebSearchInput) { } return { url, - headers: { - Accept: 'application/json', - ...(config.searxngApiKey - ? { 'X-Agent-Commons-Search-Key': config.searxngApiKey } - : {}), + init: { + headers: { + Accept: 'application/json', + ...(config.searxngApiKey + ? { 'X-Agent-Commons-Search-Key': config.searxngApiKey } + : {}), + }, }, }; } + +function buildTavilyRequest(config: WebSearchConfig, input: WebSearchInput) { + return { + url: new URL('https://api.tavily.com/search'), + init: { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + Authorization: `Bearer ${config.apiKey}`, + }, + body: JSON.stringify({ + query: input.query, + max_results: input.count, + search_depth: config.settings?.searchDepth || 'basic', + ...(input.freshness ? { time_range: input.freshness } : {}), + }), + }, + }; +} + +function buildCustomRequest(config: WebSearchConfig, input: WebSearchInput) { + const method = String(config.settings?.method || 'GET').toUpperCase(); + if (method !== 'GET' && method !== 'POST') { + throw new BadRequestException('Custom search method must be GET or POST'); + } + const url = new URL(config.endpointUrl!); + const queryField = String(config.settings?.queryField || 'q'); + const countField = String(config.settings?.countField || 'count'); + const headers: Record = { Accept: 'application/json' }; + if (config.apiKey) { + const header = String(config.settings?.apiKeyHeader || 'Authorization'); + const prefix = String(config.settings?.apiKeyPrefix ?? 'Bearer '); + headers[header] = `${prefix}${config.apiKey}`; + } + if (method === 'GET') { + url.searchParams.set(queryField, input.query); + url.searchParams.set(countField, String(input.count)); + return { url, init: { headers } }; + } + headers['Content-Type'] = 'application/json'; + return { + url, + init: { + method, + headers, + body: JSON.stringify({ + [queryField]: input.query, + [countField]: input.count, + freshness: input.freshness, + safeSearch: input.safeSearch, + }), + }, + }; +} + +function readPath(value: any, path: string) { + return path + .split('.') + .reduce((current, segment) => current?.[segment], value); +} diff --git a/apps/commons-api/src/ui-plugin/index.ts b/apps/commons-api/src/ui-plugin/index.ts new file mode 100644 index 00000000..588434c2 --- /dev/null +++ b/apps/commons-api/src/ui-plugin/index.ts @@ -0,0 +1,2 @@ +export * from './ui-plugin.module'; +export * from './ui-plugin.service'; diff --git a/apps/commons-api/src/ui-plugin/ui-plugin.controller.ts b/apps/commons-api/src/ui-plugin/ui-plugin.controller.ts new file mode 100644 index 00000000..47cc9404 --- /dev/null +++ b/apps/commons-api/src/ui-plugin/ui-plugin.controller.ts @@ -0,0 +1,73 @@ +import { + Body, + Controller, + Delete, + Get, + Param, + Put, + Query, + Req, +} from '@nestjs/common'; +import type { Request } from 'express'; +import { resolveCallerId, type ApiKeyPrincipal } from '~/modules/auth'; +import { CreateUiPluginInput, UiPluginService } from './ui-plugin.service'; + +@Controller({ version: '1', path: 'ui-plugins' }) +export class UiPluginController { + constructor(private readonly plugins: UiPluginService) {} + + @Get() + async list(@Req() request: Request, @Query('active') active?: string) { + return { + data: await this.plugins.list(requester(request).principalId, { + activeOnly: active === 'true', + }), + }; + } + + @Get('slug/:slug') + async get(@Req() request: Request, @Param('slug') slug: string) { + return { + data: await this.plugins.getBySlug(requester(request).principalId, slug), + }; + } + + @Put() + async create(@Req() request: Request, @Body() body: CreateUiPluginInput) { + const principal = requester(request); + return { + data: await this.plugins.create( + principal.principalId, + principal.workspaceId, + body, + ), + }; + } + + @Put(':pluginId/status') + async status( + @Req() request: Request, + @Param('pluginId') pluginId: string, + @Body() body: { status: 'draft' | 'active' | 'disabled' }, + ) { + return { + data: await this.plugins.setStatus( + requester(request).principalId, + pluginId, + body.status, + ), + }; + } + + @Delete(':pluginId') + remove(@Req() request: Request, @Param('pluginId') pluginId: string) { + return this.plugins.remove(requester(request).principalId, pluginId); + } +} + +function requester(request: Request) { + const principal = (request as any).principal as ApiKeyPrincipal | undefined; + const principalId = resolveCallerId(request); + if (!principalId) throw new Error('Authenticated principal required'); + return { principalId, workspaceId: principal?.workspaceId ?? null }; +} diff --git a/apps/commons-api/src/ui-plugin/ui-plugin.module.ts b/apps/commons-api/src/ui-plugin/ui-plugin.module.ts new file mode 100644 index 00000000..1febfbb7 --- /dev/null +++ b/apps/commons-api/src/ui-plugin/ui-plugin.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { UiPluginController } from './ui-plugin.controller'; +import { UiPluginService } from './ui-plugin.service'; + +@Module({ + controllers: [UiPluginController], + providers: [UiPluginService], + exports: [UiPluginService], +}) +export class UiPluginModule {} diff --git a/apps/commons-api/src/ui-plugin/ui-plugin.service.spec.ts b/apps/commons-api/src/ui-plugin/ui-plugin.service.spec.ts new file mode 100644 index 00000000..76a4dcf8 --- /dev/null +++ b/apps/commons-api/src/ui-plugin/ui-plugin.service.spec.ts @@ -0,0 +1,68 @@ +import { UiPluginService } from './ui-plugin.service'; + +describe('UiPluginService manifest boundary', () => { + const service = new UiPluginService({} as any); + + beforeEach(() => { + jest.spyOn(service as any, 'assertPublishedProject').mockResolvedValue({ + projectId: 'project-1', + workspaceId: null, + publicUrl: 'https://previews.example.com/weather/', + }); + }); + + afterEach(() => jest.restoreAllMocks()); + + it('rejects host capabilities outside the allowlist', async () => { + await expect( + service.create('user-1', null, { + name: 'Weather', + codeProjectId: 'project-1', + manifest: { + surfaces: [{ type: 'widget' }], + permissions: ['secrets.read' as any], + }, + }), + ).rejects.toThrow('Unsupported UI permission'); + }); + + it('rejects duplicate and unbounded surfaces', async () => { + await expect( + service.create('user-1', null, { + name: 'Duplicate widgets', + codeProjectId: 'project-1', + manifest: { + surfaces: [{ type: 'widget' }, { type: 'widget' }], + }, + }), + ).rejects.toThrow('Surfaces must be unique'); + }); + + it('registers generated UI as a draft with clamped widget dimensions', async () => { + const returning = jest + .fn() + .mockResolvedValue([{ pluginId: 'plugin-1', status: 'draft' }]); + const onConflictDoUpdate = jest.fn().mockReturnValue({ returning }); + const values = jest.fn().mockReturnValue({ onConflictDoUpdate }); + (service as any).db = { insert: jest.fn().mockReturnValue({ values }) }; + + await service.create('user-1', null, { + name: 'Weather', + codeProjectId: 'project-1', + manifest: { + surfaces: [{ type: 'widget', width: 2_000, height: 1 }], + permissions: ['theme.read'], + }, + }); + + expect(values).toHaveBeenCalledWith( + expect.objectContaining({ + status: 'draft', + entryUrl: 'https://previews.example.com/weather/', + manifest: expect.objectContaining({ + surfaces: [expect.objectContaining({ width: 520, height: 240 })], + }), + }), + ); + }); +}); diff --git a/apps/commons-api/src/ui-plugin/ui-plugin.service.ts b/apps/commons-api/src/ui-plugin/ui-plugin.service.ts new file mode 100644 index 00000000..42ac23ff --- /dev/null +++ b/apps/commons-api/src/ui-plugin/ui-plugin.service.ts @@ -0,0 +1,254 @@ +import { + BadRequestException, + ForbiddenException, + Injectable, + NotFoundException, +} from '@nestjs/common'; +import { and, desc, eq, sql } from 'drizzle-orm'; +import * as schema from '#/models/schema'; +import { DatabaseService } from '~/modules/database/database.service'; + +export type UiPluginPermission = 'theme.read' | 'navigation'; +export type UiPluginSurface = { + type: 'page' | 'widget'; + title?: string; + width?: number; + height?: number; +}; + +export type UiPluginManifestInput = { + schemaVersion?: '1'; + surfaces: UiPluginSurface[]; + permissions?: UiPluginPermission[]; +}; + +export type CreateUiPluginInput = { + name: string; + slug?: string; + description?: string; + version?: string; + codeProjectId: string; + manifest: UiPluginManifestInput; +}; + +const PERMISSIONS = new Set(['theme.read', 'navigation']); + +@Injectable() +export class UiPluginService { + constructor(private readonly db: DatabaseService) {} + + async list(ownerId: string, options: { activeOnly?: boolean } = {}) { + const ownerCondition = sql`lower(${schema.uiPlugin.ownerUserId}) = lower(${ownerId})`; + return this.db + .select() + .from(schema.uiPlugin) + .where( + options.activeOnly + ? and(ownerCondition, eq(schema.uiPlugin.status, 'active')) + : ownerCondition, + ) + .orderBy(desc(schema.uiPlugin.updatedAt)); + } + + async getBySlug(ownerId: string, slug: string) { + const plugin = await this.db.query.uiPlugin.findFirst({ + where: (table) => + and( + sql`lower(${table.ownerUserId}) = lower(${ownerId})`, + eq(table.slug, slug), + ), + }); + if (!plugin) throw new NotFoundException('UI plugin not found'); + return plugin; + } + + async create( + ownerId: string, + workspaceId: string | null | undefined, + input: CreateUiPluginInput, + options: { createdByAgentId?: string; status?: 'draft' | 'active' } = {}, + ) { + const project = await this.assertPublishedProject( + ownerId, + input.codeProjectId, + ); + const manifest = normalizeManifest(input.manifest); + const name = input.name?.trim().slice(0, 100); + if (!name) throw new BadRequestException('Plugin name is required'); + const slug = slugify(input.slug || name); + if (!slug) throw new BadRequestException('Plugin slug is required'); + const values = { + ownerUserId: ownerId, + workspaceId: workspaceId ?? project.workspaceId, + createdByAgentId: options.createdByAgentId ?? null, + codeProjectId: project.projectId, + name, + slug, + description: input.description?.trim().slice(0, 1_000) || null, + version: normalizeVersion(input.version), + entryUrl: project.publicUrl, + manifest, + status: options.status ?? ('draft' as const), + updatedAt: new Date(), + }; + const [plugin] = await this.db + .insert(schema.uiPlugin) + .values(values) + .onConflictDoUpdate({ + target: [schema.uiPlugin.ownerUserId, schema.uiPlugin.slug], + set: values, + }) + .returning(); + return plugin; + } + + async createForAgent(agentId: string, input: CreateUiPluginInput) { + const agent = await this.db.query.agent.findFirst({ + where: (table) => eq(table.agentId, agentId), + columns: { + ownerUserId: true, + owner: true, + workspaceId: true, + }, + }); + const ownerId = agent?.ownerUserId ?? agent?.owner; + if (!agent || !ownerId) + throw new ForbiddenException('Agent owner is required'); + return this.create(ownerId, agent.workspaceId, input, { + createdByAgentId: agentId, + status: 'draft', + }); + } + + async setStatus( + ownerId: string, + pluginId: string, + status: 'draft' | 'active' | 'disabled', + ) { + if (!['draft', 'active', 'disabled'].includes(status)) { + throw new BadRequestException('Invalid UI plugin status'); + } + const [plugin] = await this.db + .update(schema.uiPlugin) + .set({ status, updatedAt: new Date() }) + .where( + and( + eq(schema.uiPlugin.pluginId, pluginId), + sql`lower(${schema.uiPlugin.ownerUserId}) = lower(${ownerId})`, + ), + ) + .returning(); + if (!plugin) throw new NotFoundException('UI plugin not found'); + return plugin; + } + + async remove(ownerId: string, pluginId: string) { + const removed = await this.db + .delete(schema.uiPlugin) + .where( + and( + eq(schema.uiPlugin.pluginId, pluginId), + sql`lower(${schema.uiPlugin.ownerUserId}) = lower(${ownerId})`, + ), + ) + .returning({ pluginId: schema.uiPlugin.pluginId }); + if (!removed.length) throw new NotFoundException('UI plugin not found'); + return { deleted: true }; + } + + private async assertPublishedProject(ownerId: string, projectId: string) { + const [row] = await this.db + .select({ + projectId: schema.codeProject.projectId, + workspaceId: schema.codeProject.workspaceId, + publicUrl: schema.codeProjectDeployment.publicUrl, + deploymentStatus: schema.codeProjectDeployment.status, + }) + .from(schema.codeProject) + .leftJoin( + schema.codeProjectDeployment, + eq( + schema.codeProjectDeployment.deploymentId, + schema.codeProject.latestDeploymentId, + ), + ) + .where( + and( + eq(schema.codeProject.projectId, projectId), + sql`lower(${schema.codeProject.ownerUserId}) = lower(${ownerId})`, + ), + ) + .limit(1); + if (!row) throw new NotFoundException('Code project not found'); + if (row.deploymentStatus !== 'ready' || !row.publicUrl) { + throw new BadRequestException( + 'Publish and verify the code project before registering it as UI', + ); + } + return { ...row, publicUrl: row.publicUrl }; + } +} + +function normalizeManifest(input: UiPluginManifestInput) { + if (!input || !Array.isArray(input.surfaces) || !input.surfaces.length) { + throw new BadRequestException('At least one UI surface is required'); + } + if (input.surfaces.length > 2) { + throw new BadRequestException('A plugin supports at most two surfaces'); + } + const seen = new Set(); + const surfaces = input.surfaces.map((surface) => { + if (!['page', 'widget'].includes(surface.type) || seen.has(surface.type)) { + throw new BadRequestException( + 'Surfaces must be unique page or widget entries', + ); + } + seen.add(surface.type); + return { + type: surface.type, + title: surface.title?.trim().slice(0, 80) || undefined, + ...(surface.type === 'widget' + ? { + width: clamp(surface.width, 280, 520, 380), + height: clamp(surface.height, 240, 720, 480), + } + : {}), + }; + }); + const permissions = [...new Set(input.permissions ?? [])]; + for (const permission of permissions) { + if (!PERMISSIONS.has(permission)) { + throw new BadRequestException(`Unsupported UI permission ${permission}`); + } + } + return { schemaVersion: '1' as const, surfaces, permissions }; +} + +function clamp( + value: number | undefined, + minimum: number, + maximum: number, + fallback: number, +) { + if (!Number.isFinite(value)) return fallback; + return Math.min(maximum, Math.max(minimum, Math.round(value as number))); +} + +function slugify(value: string) { + return value + .toLowerCase() + .trim() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80); +} + +function normalizeVersion(value?: string) { + const version = (value || '1.0.0').trim(); + if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)) { + throw new BadRequestException( + 'Plugin version must use semantic versioning', + ); + } + return version; +} diff --git a/apps/commons-api/src/wallet/dto/wallet.dto.ts b/apps/commons-api/src/wallet/dto/wallet.dto.ts index be163b4a..17f40a3a 100644 --- a/apps/commons-api/src/wallet/dto/wallet.dto.ts +++ b/apps/commons-api/src/wallet/dto/wallet.dto.ts @@ -13,14 +13,16 @@ export interface CreateWalletDto { export interface WalletBalanceDto { address: string; chainId: string; - usdc: string; // formatted USDC balance (6 decimals) - native: string; // formatted native token balance (ETH) + usdc: string; // formatted USDC balance (6 decimals) + native: string; // formatted native token balance (ETH) } export interface WalletResponseDto { id: string; agentId: string; walletType: WalletType; + provider?: string; + providerWalletId?: string | null; address: string; smartAccountAddress?: string | null; chainId: string; diff --git a/apps/commons-api/src/wallet/wallet.module.ts b/apps/commons-api/src/wallet/wallet.module.ts index 2073d94d..408a7999 100644 --- a/apps/commons-api/src/wallet/wallet.module.ts +++ b/apps/commons-api/src/wallet/wallet.module.ts @@ -2,9 +2,10 @@ import { Module } from '@nestjs/common'; import { WalletService } from './wallet.service'; import { WalletController } from './wallet.controller'; import { EncryptionModule } from '~/modules/encryption'; +import { CapabilityProviderModule } from '~/provider'; @Module({ - imports: [EncryptionModule], + imports: [EncryptionModule, CapabilityProviderModule], controllers: [WalletController], providers: [WalletService], exports: [WalletService], diff --git a/apps/commons-api/src/wallet/wallet.service.ts b/apps/commons-api/src/wallet/wallet.service.ts index eebe0de8..8f529907 100644 --- a/apps/commons-api/src/wallet/wallet.service.ts +++ b/apps/commons-api/src/wallet/wallet.service.ts @@ -9,20 +9,33 @@ import { DatabaseService } from '~/modules/database/database.service'; import { EncryptionService } from '~/modules/encryption'; import * as schema from '#/models/schema'; import { eq, and } from 'drizzle-orm'; -import { createPublicClient, createWalletClient, http, formatUnits, parseUnits, encodeFunctionData } from 'viem'; +import { + createPublicClient, + createWalletClient, + http, + formatUnits, + parseUnits, + encodeFunctionData, +} from 'viem'; import { generatePrivateKey, privateKeyToAccount } from 'viem/accounts'; import { baseSepolia } from '#/lib/baseSepolia'; import { safeFetch } from '~/utils/safe-fetch'; -import type { CreateWalletDto, WalletBalanceDto, WalletResponseDto } from './dto/wallet.dto'; +import type { + CreateWalletDto, + WalletBalanceDto, + WalletResponseDto, +} from './dto/wallet.dto'; +import { CapabilityProviderService } from '~/provider'; export interface TransferDto { toAddress: string; - amount: string; // human-readable e.g. "10.5" + amount: string; // human-readable e.g. "10.5" tokenSymbol?: 'USDC' | 'ETH'; } /** Base Sepolia USDC contract address */ -const USDC_ADDRESS_BASE_SEPOLIA = '0x036CbD53842c5426634e7929541eC2318f3dCF7e' as const; +const USDC_ADDRESS_BASE_SEPOLIA = + '0x036CbD53842c5426634e7929541eC2318f3dCF7e' as const; const ERC20_BALANCE_ABI = [ { @@ -45,6 +58,7 @@ export class WalletService { constructor( private db: DatabaseService, private encryption: EncryptionService, + private capabilityProviders: CapabilityProviderService, ) {} /** @@ -54,14 +68,64 @@ export class WalletService { * - 'erc4337': placeholder — session key flow to be implemented with ZeroDev */ async createWallet(dto: CreateWalletDto): Promise { - const { agentId, walletType = 'eoa', label = 'Primary', chainId = '84532' } = dto; + const { + agentId, + walletType = 'eoa', + label = 'Primary', + chainId = '84532', + } = dto; + + const agent = await this.db.query.agent.findFirst({ + where: (table) => eq(table.agentId, agentId), + columns: { ownerUserId: true, owner: true }, + }); + const ownerId = agent?.ownerUserId ?? agent?.owner; + const configured = ownerId + ? await this.capabilityProviders.resolve(ownerId, 'wallet') + : null; + + if (configured?.provider === 'custom') { + const remote = await this.customWalletRequest<{ + walletId?: string; + id?: string; + address: string; + smartAccountAddress?: string; + }>(configured, 'POST', '/wallets', { agentId, label, chainId }); + if (!remote.address) { + throw new BadRequestException( + 'Custom wallet provider did not return an address', + ); + } + const [wallet] = await this.db + .insert(schema.agentWallet) + .values({ + agentId, + walletType: 'external', + provider: 'custom', + providerWalletId: remote.walletId ?? remote.id ?? remote.address, + address: remote.address.toLowerCase(), + smartAccountAddress: remote.smartAccountAddress ?? null, + chainId, + label, + isActive: true, + }) + .returning(); + return this.toResponse(wallet); + } + if (configured?.provider === 'external' && walletType !== 'external') { + throw new BadRequestException( + 'The selected wallet provider requires an owner-connected external address', + ); + } let address: string; let encryptedPrivateKey: string | undefined; if (walletType === 'external') { if (!dto.externalAddress) { - throw new BadRequestException('externalAddress is required for external wallets'); + throw new BadRequestException( + 'externalAddress is required for external wallets', + ); } address = dto.externalAddress.toLowerCase(); } else if (walletType === 'eoa' || walletType === 'erc4337') { @@ -79,6 +143,7 @@ export class WalletService { .values({ agentId, walletType, + provider: configured?.provider ?? 'commons_mpc', address, encryptedPrivateKey: encryptedPrivateKey ?? null, chainId, @@ -149,6 +214,17 @@ export class WalletService { }); if (!wallet) throw new NotFoundException(`Wallet ${walletId} not found`); + if (wallet.provider === 'custom') { + const configured = await this.customProviderForWallet(wallet); + return this.customWalletRequest( + configured, + 'GET', + `/wallets/${encodeURIComponent( + wallet.providerWalletId ?? wallet.address, + )}/balance`, + ); + } + const address = wallet.address as `0x${string}`; const [nativeBalance, usdcBalance] = await Promise.all([ @@ -173,16 +249,34 @@ export class WalletService { * Transfer USDC (or native ETH) from an EOA wallet to another address. * The wallet must have an encrypted private key stored (EOA type). */ - async transfer(walletId: string, dto: TransferDto): Promise<{ txHash: string }> { + async transfer( + walletId: string, + dto: TransferDto, + ): Promise<{ txHash: string }> { const wallet = await this.db.query.agentWallet.findFirst({ where: (w) => eq(w.id, walletId), }); if (!wallet) throw new NotFoundException(`Wallet ${walletId} not found`); + if (wallet.provider === 'custom') { + const configured = await this.customProviderForWallet(wallet); + return this.customWalletRequest<{ txHash: string }>( + configured, + 'POST', + `/wallets/${encodeURIComponent( + wallet.providerWalletId ?? wallet.address, + )}/transfer`, + dto, + ); + } if (!wallet.encryptedPrivateKey) { - throw new BadRequestException('This wallet has no stored private key — only EOA wallets can send transactions'); + throw new BadRequestException( + 'This wallet has no stored private key — only EOA wallets can send transactions', + ); } - const privateKey = this.decryptKey(wallet.encryptedPrivateKey) as `0x${string}`; + const privateKey = this.decryptKey( + wallet.encryptedPrivateKey, + ) as `0x${string}`; const account = privateKeyToAccount(privateKey); const to = dto.toAddress as `0x${string}`; const tokenSymbol = dto.tokenSymbol ?? 'USDC'; @@ -281,7 +375,9 @@ export class WalletService { ); } - const privateKey = this.decryptKey(wallet.encryptedPrivateKey) as `0x${string}`; + const privateKey = this.decryptKey( + wallet.encryptedPrivateKey, + ) as `0x${string}`; const account = privateKeyToAccount(privateKey); // Build a viem wallet client for x402 signing @@ -293,7 +389,10 @@ export class WalletService { // Select the first matching payment requirement (prefer exact/base-sepolia) // eslint-disable-next-line @typescript-eslint/no-require-imports - const { createPaymentHeader, selectPaymentRequirements } = require('x402/client'); + const { + createPaymentHeader, + selectPaymentRequirements, + } = require('x402/client'); const requirements = selectPaymentRequirements(accepts); if (!requirements) { throw new Error('x402: no supported payment requirement in 402 response'); @@ -303,7 +402,11 @@ export class WalletService { `x402: paying ${requirements.maxAmountRequired} ${requirements.asset} on ${requirements.network} for ${agentId}`, ); - const paymentHeader = await createPaymentHeader(viemWalletClient, 1, requirements); + const paymentHeader = await createPaymentHeader( + viemWalletClient, + 1, + requirements, + ); // Retry with payment header const retryRes = await safeFetch(url, { @@ -330,11 +433,66 @@ export class WalletService { return this.encryption.decrypt(encryptedValue, iv, tag); } - private toResponse(wallet: typeof schema.agentWallet.$inferSelect): WalletResponseDto { + private async customProviderForWallet( + wallet: typeof schema.agentWallet.$inferSelect, + ) { + const agent = await this.db.query.agent.findFirst({ + where: (table) => eq(table.agentId, wallet.agentId), + columns: { ownerUserId: true, owner: true }, + }); + const ownerId = agent?.ownerUserId ?? agent?.owner; + const configured = ownerId + ? await this.capabilityProviders.resolve(ownerId, 'wallet') + : null; + if (!configured || configured.provider !== 'custom') { + throw new BadRequestException( + 'The custom wallet adapter is no longer configured', + ); + } + return configured; + } + + private async customWalletRequest( + provider: Awaited> & {}, + method: 'GET' | 'POST', + path: string, + body?: unknown, + ): Promise { + const endpoint = provider?.endpointUrl?.replace(/\/$/, ''); + if (!endpoint) + throw new BadRequestException('Custom wallet endpoint is missing'); + const response = await fetch(`${endpoint}${path}`, { + method, + headers: { + Accept: 'application/json', + ...(provider.credentials.apiKey + ? { Authorization: `Bearer ${provider.credentials.apiKey}` } + : {}), + ...(body !== undefined ? { 'Content-Type': 'application/json' } : {}), + }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(20_000), + }); + const payload: any = await response.json().catch(() => null); + if (!response.ok) { + throw new BadRequestException( + payload?.message || + payload?.error || + `Wallet provider returned ${response.status}`, + ); + } + return payload as T; + } + + private toResponse( + wallet: typeof schema.agentWallet.$inferSelect, + ): WalletResponseDto { return { id: wallet.id, agentId: wallet.agentId, walletType: wallet.walletType as any, + provider: wallet.provider, + providerWalletId: wallet.providerWalletId, address: wallet.address, smartAccountAddress: wallet.smartAccountAddress, chainId: wallet.chainId, diff --git a/apps/commons-app/app/api/providers/[capability]/route.ts b/apps/commons-app/app/api/providers/[capability]/route.ts new file mode 100644 index 00000000..145a1286 --- /dev/null +++ b/apps/commons-app/app/api/providers/[capability]/route.ts @@ -0,0 +1,50 @@ +import { NextRequest, NextResponse } from "next/server"; +import { backendAuthHeaders } from "@/lib/api-headers"; + +const baseUrl = process.env.NEXT_PUBLIC_NEST_API_BASE_URL; + +export async function PUT( + request: NextRequest, + { params }: { params: Promise<{ capability: string }> } +) { + if (!baseUrl) { + return NextResponse.json( + { error: "Server base URL not configured" }, + { status: 500 } + ); + } + const { capability } = await params; + const body = await request.json(); + const response = await fetch( + `${baseUrl}/v1/providers/${encodeURIComponent(capability)}`, + { + method: "PUT", + headers: { + "Content-Type": "application/json", + ...(await backendAuthHeaders()), + }, + body: JSON.stringify(body), + } + ); + const payload = await response.json().catch(() => ({})); + return NextResponse.json(payload, { status: response.status }); +} + +export async function DELETE( + _request: NextRequest, + { params }: { params: Promise<{ capability: string }> } +) { + if (!baseUrl) { + return NextResponse.json( + { error: "Server base URL not configured" }, + { status: 500 } + ); + } + const { capability } = await params; + const response = await fetch( + `${baseUrl}/v1/providers/${encodeURIComponent(capability)}`, + { method: "DELETE", headers: await backendAuthHeaders() } + ); + const payload = await response.json().catch(() => ({})); + return NextResponse.json(payload, { status: response.status }); +} diff --git a/apps/commons-app/app/api/providers/route.ts b/apps/commons-app/app/api/providers/route.ts new file mode 100644 index 00000000..918fe8ba --- /dev/null +++ b/apps/commons-app/app/api/providers/route.ts @@ -0,0 +1,29 @@ +import { NextResponse } from "next/server"; +import { backendAuthHeaders } from "@/lib/api-headers"; + +const baseUrl = process.env.NEXT_PUBLIC_NEST_API_BASE_URL; + +export async function GET() { + if (!baseUrl) { + return NextResponse.json( + { error: "Server base URL not configured" }, + { status: 500 } + ); + } + try { + const response = await fetch(`${baseUrl}/v1/providers`, { + cache: "no-store", + headers: await backendAuthHeaders(), + }); + const payload = await response.json().catch(() => ({})); + return NextResponse.json(payload, { status: response.status }); + } catch (error) { + return NextResponse.json( + { + error: + error instanceof Error ? error.message : "Could not load providers", + }, + { status: 500 } + ); + } +} diff --git a/apps/commons-app/app/api/skills/[skillId]/agents/[agentId]/route.ts b/apps/commons-app/app/api/skills/[skillId]/agents/[agentId]/route.ts new file mode 100644 index 00000000..c7aab687 --- /dev/null +++ b/apps/commons-app/app/api/skills/[skillId]/agents/[agentId]/route.ts @@ -0,0 +1,12 @@ +import { proxyBackend } from "@/lib/backend-proxy"; + +export async function PUT( + request: Request, + { params }: { params: Promise<{ skillId: string; agentId: string }> }, +) { + const { skillId, agentId } = await params; + return proxyBackend( + `/v1/skills/${encodeURIComponent(skillId)}/agents/${encodeURIComponent(agentId)}`, + { method: "PUT", body: await request.json() }, + ); +} diff --git a/apps/commons-app/app/api/skills/agents/[agentId]/route.ts b/apps/commons-app/app/api/skills/agents/[agentId]/route.ts new file mode 100644 index 00000000..1af0b06e --- /dev/null +++ b/apps/commons-app/app/api/skills/agents/[agentId]/route.ts @@ -0,0 +1,9 @@ +import { proxyBackend } from "@/lib/backend-proxy"; + +export async function GET( + _request: Request, + { params }: { params: Promise<{ agentId: string }> }, +) { + const { agentId } = await params; + return proxyBackend(`/v1/skills/agents/${encodeURIComponent(agentId)}`); +} diff --git a/apps/commons-app/app/api/skills/import/route.ts b/apps/commons-app/app/api/skills/import/route.ts new file mode 100644 index 00000000..3591420d --- /dev/null +++ b/apps/commons-app/app/api/skills/import/route.ts @@ -0,0 +1,31 @@ +import { NextRequest, NextResponse } from "next/server"; +import { backendAuthHeaders } from "@/lib/api-headers"; + +const baseUrl = process.env.NEXT_PUBLIC_NEST_API_BASE_URL; + +export async function POST(request: NextRequest) { + if (!baseUrl) { + return NextResponse.json( + { error: "Server base URL not configured" }, + { status: 500 } + ); + } + try { + const body = await request.formData(); + const response = await fetch(`${baseUrl}/v1/skills/import`, { + method: "POST", + headers: await backendAuthHeaders(), + body, + }); + const payload = await response.json().catch(() => ({})); + return NextResponse.json(payload, { status: response.status }); + } catch (error) { + return NextResponse.json( + { + error: + error instanceof Error ? error.message : "Could not import skill", + }, + { status: 500 } + ); + } +} diff --git a/apps/commons-app/app/api/ui-plugins/[pluginId]/route.ts b/apps/commons-app/app/api/ui-plugins/[pluginId]/route.ts new file mode 100644 index 00000000..6ba51cdd --- /dev/null +++ b/apps/commons-app/app/api/ui-plugins/[pluginId]/route.ts @@ -0,0 +1,29 @@ +import { NextRequest, NextResponse } from "next/server"; +import { backendAuthHeaders } from "@/lib/api-headers"; + +const baseUrl = process.env.NEXT_PUBLIC_NEST_API_BASE_URL; + +export async function DELETE( + _request: NextRequest, + { params }: { params: Promise<{ pluginId: string }> } +) { + if (!baseUrl) return unavailable(); + const { pluginId } = await params; + const response = await fetch( + `${baseUrl}/v1/ui-plugins/${encodeURIComponent(pluginId)}`, + { method: "DELETE", headers: await backendAuthHeaders() } + ); + return forward(response); +} + +async function forward(response: Response) { + const payload = await response.json().catch(() => ({})); + return NextResponse.json(payload, { status: response.status }); +} + +function unavailable() { + return NextResponse.json( + { error: "Server base URL not configured" }, + { status: 500 } + ); +} diff --git a/apps/commons-app/app/api/ui-plugins/[pluginId]/status/route.ts b/apps/commons-app/app/api/ui-plugins/[pluginId]/status/route.ts new file mode 100644 index 00000000..c900bc30 --- /dev/null +++ b/apps/commons-app/app/api/ui-plugins/[pluginId]/status/route.ts @@ -0,0 +1,36 @@ +import { NextRequest, NextResponse } from "next/server"; +import { backendAuthHeaders } from "@/lib/api-headers"; + +const baseUrl = process.env.NEXT_PUBLIC_NEST_API_BASE_URL; + +export async function PUT( + request: NextRequest, + { params }: { params: Promise<{ pluginId: string }> } +) { + if (!baseUrl) return unavailable(); + const { pluginId } = await params; + const response = await fetch( + `${baseUrl}/v1/ui-plugins/${encodeURIComponent(pluginId)}/status`, + { + method: "PUT", + headers: { + "Content-Type": "application/json", + ...(await backendAuthHeaders()), + }, + body: JSON.stringify(await request.json()), + } + ); + return forward(response); +} + +async function forward(response: Response) { + const payload = await response.json().catch(() => ({})); + return NextResponse.json(payload, { status: response.status }); +} + +function unavailable() { + return NextResponse.json( + { error: "Server base URL not configured" }, + { status: 500 } + ); +} diff --git a/apps/commons-app/app/api/ui-plugins/route.ts b/apps/commons-app/app/api/ui-plugins/route.ts new file mode 100644 index 00000000..a20692d9 --- /dev/null +++ b/apps/commons-app/app/api/ui-plugins/route.ts @@ -0,0 +1,41 @@ +import { NextRequest, NextResponse } from "next/server"; +import { backendAuthHeaders } from "@/lib/api-headers"; + +const baseUrl = process.env.NEXT_PUBLIC_NEST_API_BASE_URL; + +export async function GET(request: NextRequest) { + if (!baseUrl) return unavailable(); + const active = request.nextUrl.searchParams.get("active"); + const response = await fetch( + `${baseUrl}/v1/ui-plugins${ + active ? `?active=${encodeURIComponent(active)}` : "" + }`, + { cache: "no-store", headers: await backendAuthHeaders() } + ); + return forward(response); +} + +export async function PUT(request: NextRequest) { + if (!baseUrl) return unavailable(); + const response = await fetch(`${baseUrl}/v1/ui-plugins`, { + method: "PUT", + headers: { + "Content-Type": "application/json", + ...(await backendAuthHeaders()), + }, + body: JSON.stringify(await request.json()), + }); + return forward(response); +} + +async function forward(response: Response) { + const payload = await response.json().catch(() => ({})); + return NextResponse.json(payload, { status: response.status }); +} + +function unavailable() { + return NextResponse.json( + { error: "Server base URL not configured" }, + { status: 500 } + ); +} diff --git a/apps/commons-app/app/api/ui-plugins/slug/[slug]/route.ts b/apps/commons-app/app/api/ui-plugins/slug/[slug]/route.ts new file mode 100644 index 00000000..5bc81010 --- /dev/null +++ b/apps/commons-app/app/api/ui-plugins/slug/[slug]/route.ts @@ -0,0 +1,23 @@ +import { NextResponse } from "next/server"; +import { backendAuthHeaders } from "@/lib/api-headers"; + +const baseUrl = process.env.NEXT_PUBLIC_NEST_API_BASE_URL; + +export async function GET( + _request: Request, + { params }: { params: Promise<{ slug: string }> } +) { + if (!baseUrl) { + return NextResponse.json( + { error: "Server base URL not configured" }, + { status: 500 } + ); + } + const { slug } = await params; + const response = await fetch( + `${baseUrl}/v1/ui-plugins/slug/${encodeURIComponent(slug)}`, + { cache: "no-store", headers: await backendAuthHeaders() } + ); + const payload = await response.json().catch(() => ({})); + return NextResponse.json(payload, { status: response.status }); +} diff --git a/apps/commons-app/app/apps/[slug]/page.tsx b/apps/commons-app/app/apps/[slug]/page.tsx new file mode 100644 index 00000000..4342fd84 --- /dev/null +++ b/apps/commons-app/app/apps/[slug]/page.tsx @@ -0,0 +1,79 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useParams } from "next/navigation"; +import Link from "next/link"; +import { ArrowLeft, Loader2, ShieldAlert } from "lucide-react"; +import { PluginFrame } from "@/components/plugins/plugin-frame"; +import type { UiPlugin } from "@/components/plugins/types"; + +export default function CustomAppPage() { + const { slug } = useParams<{ slug: string }>(); + const [plugin, setPlugin] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + fetch(`/api/ui-plugins/slug/${encodeURIComponent(slug)}`, { + cache: "no-store", + }) + .then(async (response) => { + const payload = await response.json().catch(() => ({})); + if (!response.ok) throw new Error(payload.message || "App not found"); + if (payload.data.status !== "active") { + throw new Error("This custom app has not been enabled."); + } + setPlugin(payload.data); + }) + .catch((reason) => + setError(reason instanceof Error ? reason.message : "App not found") + ); + }, [slug]); + + if (error) { + return ( +
+
+ +

Custom app unavailable

+

{error}

+ + Back to Studio Apps + +
+
+ ); + } + if (!plugin) { + return ( +
+ +
+ ); + } + return ( +
+
+ + + +
+

{plugin.name}

+

+ Sandboxed custom app · v{plugin.version} +

+
+
+ +
+ ); +} diff --git a/apps/commons-app/app/layout.tsx b/apps/commons-app/app/layout.tsx index 6e63cb55..58a6428b 100644 --- a/apps/commons-app/app/layout.tsx +++ b/apps/commons-app/app/layout.tsx @@ -12,6 +12,7 @@ import { FloatingCommonsCopilot } from "@/components/copilot/floating-commons-co import { auth } from "@/auth"; import type { Session } from "next-auth"; import { getAppBaseUrl } from "@/lib/app-url"; +import { PluginWidgetHost } from "@/components/plugins/plugin-widget-host"; const spaceGrotesk = Space_Grotesk({ weight: ["400", "500", "600", "700"], @@ -107,6 +108,7 @@ export default async function RootLayout({ {children} + diff --git a/apps/commons-app/app/library/page.tsx b/apps/commons-app/app/library/page.tsx index 9eb90962..282418db 100644 --- a/apps/commons-app/app/library/page.tsx +++ b/apps/commons-app/app/library/page.tsx @@ -99,6 +99,7 @@ export default function LibraryPage() { const [loading, setLoading] = useState(true); const [uploading, setUploading] = useState(false); const [error, setError] = useState(""); + const [notice, setNotice] = useState(""); const [query, setQuery] = useState(""); const [view, setView] = useState("all"); const [source, setSource] = useState("all"); @@ -170,6 +171,7 @@ export default function LibraryPage() { return; setUploading(true); setError(""); + setNotice(""); try { const body = new FormData(); [...files].forEach((file) => body.append("files", file)); @@ -181,6 +183,17 @@ export default function LibraryPage() { const data = await response.json(); if (!response.ok) throw new Error(data?.message || data?.error || "Upload failed"); + const uploaded = Array.isArray(data?.data) ? data.data : []; + const reused = uploaded.filter( + (item: { reused?: boolean }) => item.reused, + ); + if (reused.length) { + setNotice( + reused.length === uploaded.length + ? `${reused.length === 1 ? "This file was" : "These files were"} already in your Library, so the existing ${reused.length === 1 ? "item was" : "items were"} reused.` + : `${reused.length} duplicate ${reused.length === 1 ? "file was" : "files were"} reused from your Library; only new files were uploaded.`, + ); + } await load(); } catch (cause) { setError(cause instanceof Error ? cause.message : "Upload failed"); @@ -411,6 +424,11 @@ export default function LibraryPage() { {error} )} + {notice && ( +
+ {notice} +
+ )} {loading ? (
diff --git a/apps/commons-app/app/studio/[tab]/page.tsx b/apps/commons-app/app/studio/[tab]/page.tsx index 3fe90404..086dbe5a 100644 --- a/apps/commons-app/app/studio/[tab]/page.tsx +++ b/apps/commons-app/app/studio/[tab]/page.tsx @@ -17,6 +17,7 @@ import { CreateWorkflowDialog } from "@/components/workflows/create-workflow-dia import { CreateToolDialog } from "@/components/tools/create-tool-dialog"; import { TaskManagementView } from "@/components/tasks/task-management-view"; import { SkillsMarketplaceView } from "@/components/skills/skills-marketplace-view"; +import { UiPluginsView } from "@/components/plugins/ui-plugins-view"; import { AlertCircle, Loader2 } from "lucide-react"; import { useAuth } from "@/context/AuthContext"; import { CreateButton, PageHeader } from "@/components/layout/page-header"; @@ -52,9 +53,7 @@ const StudioPage: NextPage = () => { loading: loadingAgents, error: agentsError, refresh: refreshAgents, - } = useAgents( - activeTab === "agents" ? userAddress : undefined, - ); + } = useAgents(activeTab === "agents" ? userAddress : undefined); // Agents arrive ordered by latest interaction (falling back to creation); // we page through them client-side, 10 floating profiles at a time. @@ -62,7 +61,9 @@ const StudioPage: NextPage = () => { const [agentPageSize, setAgentPageSize] = useState(10); useEffect(() => { - const stored = Number(window.localStorage.getItem("studio-agents-per-page")); + const stored = Number( + window.localStorage.getItem("studio-agents-per-page") + ); if (AGENT_PAGE_SIZES.includes(stored)) setAgentPageSize(stored); }, []); @@ -80,11 +81,8 @@ const StudioPage: NextPage = () => { const pagedAgents = useMemo( () => - agents.slice( - agentPage * agentPageSize, - (agentPage + 1) * agentPageSize, - ), - [agents, agentPage, agentPageSize], + agents.slice(agentPage * agentPageSize, (agentPage + 1) * agentPageSize), + [agents, agentPage, agentPageSize] ); const mainContent = useMemo(() => { @@ -121,6 +119,12 @@ const StudioPage: NextPage = () => { />
); + case "apps": + return ( +
+ +
+ ); // Only the real /studio/agents route mounts the launcher — it depends on // the AgentProvider from that route's layout. The default fallback (for // unknown /studio/ segments served without that provider) renders just @@ -136,9 +140,12 @@ const StudioPage: NextPage = () => { ) : agentsError ? (
-

Couldn’t load your agents

+

+ Couldn’t load your agents +

- Your account is still signed in. The connection to Agent Commons was interrupted. + Your account is still signed in. The connection to Agent + Commons was interrupted.

- } - > - {loading ? ( - - ) : skills.length === 0 ? ( -
- No skills configured for this agent yet. -
- ) : ( -
- {skills.map((skill) => ( +
+
+
+ + setSearch(event.target.value)} + placeholder="Search available skills" + className="h-9 pl-9" + /> +
+
+
+ {(["assigned", "all"] as const).map((value) => ( ))}
- )} - - -
-
- - - setDraft((d) => ({ ...d, name: e.target.value })) - } - /> -
-
- -