diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index 46732fca88..a7949fbf17 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -37,7 +37,7 @@ We collect the following categories of events: | `error_recovered` | Successful recovery from a transient error (error type, strategy, attempt count) | | `mcp_server_census` | MCP server capabilities after connect (tool and resource counts, but no tool names) | | `context_overflow_recovered` | Context overflow is handled (strategy) | -| `skill_used` | A skill is loaded (skill name, source — `builtin`, `global`, or `project`, and trigger — `user`, `auto`, or `suggestion` — no skill content) | +| `skill_used` | A skill is loaded (skill name, source — `builtin`, `global`, `project`, or `workspace`, and trigger — `user`, `auto`, or `suggestion` — no skill content) | | `plan_revision` | A plan revision occurs in Plan mode (revision_number, action: `refine`, `approve`, `reject`, or `cap_reached`) | | `feature_suggestion` | A post-connection feature suggestion is shown (suggestion_type, suggestions_shown, warehouse_type — no user input) | | `sql_execute_failure` | A SQL execution fails (warehouse type, query type, error message, PII-masked SQL — no raw values) | @@ -77,6 +77,37 @@ We collect the following categories of events: Each event includes a timestamp, session ID, a per-launch correlation ID (`launch_id` — a random value regenerated every process start, not persisted and not derived from your machine or identity; it exists only to group events from the same run), CLI version, and a machine ID (a random UUID stored in `~/.altimate/machine-id`, generated once and not derived from your hardware or identity). When the machine is signed in to Altimate, each event also carries `user_id` and `tenant`, described under [User Identification](#new-user-identification). +### Workspace events + +These events describe projects linked to an Altimate workspace. A workspace is identified only by its numeric id (`workspace_id`, or `to_workspace_id` and `from_workspace_id` on a switch). No workspace name, project path, repository remote, skill content, memory content or prompt text is sent. + +| Event | Description | +|-------|-------------| +| `workspace_linked` | A project was linked to a workspace and the server confirmed it. `how` is `bind_existing`, `create_and_bind`, `rebind_remote` or `rebind_path`. | +| `workspace_created` | A workspace was created from the CLI or the TUI. `how` is `create_and_bind`, or `create_unbound` when the link follows as a separate step. | +| `workspace_link_failed` | A link step failed. `how` names the step, and `reason` is a class: `auth`, `not_found`, `network`, `conflict` or `other`. No error text. | +| `workspace_unlinked` | A project was unlinked, with the workspace id when the CLI knew it. | +| `workspace_switched` | A linked project was moved to another workspace. `how` is `rebind_remote` or `rebind_path`. | +| `workspace_auth_expired` | A workspace call was answered 401: the sign-in has expired. Carries `endpoint` and `status`. | +| `workspace_api_error` | A workspace call failed. `status_class` is `4xx`, `5xx`, `network`, `timeout`, or `response` (a success status with an empty or unreadable body), plus the HTTP `status` when there was one. A 403 is reported here, not as expired auth. A status the caller handles as an answer is not a failure and is not reported, such as a 404 from the link lookups ("not linked") or from an unlink ("already unlinked"). `GET /users/me` is not a workspace call and is not reported. | +| `starter_shown` | The "what this workspace provides" starter was served after a link: to the TUI, to the IDE extension's chat, or printed by `altimate-code link`. It counts the starter being served, so a client that closes before rendering it is still counted. Contains the counts of skills and integrations when known, `knowledge` (`off`, `all`, `selected` or `unknown`) and `memory_enabled`. | +| `starter_action_picked` | The user chose an option from the TUI starter: `action` is `prompt`, `open` or `done`, with `prompt_index` (its position) for a suggested prompt. Never the prompt's text. | +| `workspace_memory_contributed` | A memory block was written back to the workspace's shared memory, including the existing blocks seeded when a project is linked. `block_count` is per write. No memory content. | +| `workspace_sync_failed` | Syncing the workspace's skills or memory to this project changed to failing. `kind` is `skills` or `memory`, and `reason` is a class. Sent once per change, not on every retry. | +| `workspace_engine_attach` | A session's connection to the workspace integration engine settled on an outcome (`attached`, `engine_missing`, `engine_too_old`, `connect_failed`, `disabled` or `unbound`), with the engine version and counts of integrations. Sent when the outcome changes, not every turn. | +| `workspace_engine_install_prompt` | The offer to install or update the engine: `action` is `shown`, `accepted`, `installed`, `install_failed`, `copied` or `declined`, with the `reason` and whether Node.js was found and its major version. | +| `workspace_engine_precedence` | Whether the workspace's integrations take precedence over the project's own connections, with counts of shadowed connection types and connections. Sent when the state changes. | +| `workspace_engine_redirect` | A `datamate_manager` call was refused because the project is linked to a workspace. Contains the tool and operation names. | +| `workspace_engine_error` | The engine failed at a stage (`attach`, `install` or `precedence`). `message` is masked (paths, credentials and quoted strings are replaced) and capped at 200 characters. | + +Other rows gain workspace fields in a linked project: + +- Every event carries `client: "altimate-code"`, which names the product that sent it. `source` still says how the CLI was launched. +- Rows of a turn in a linked project carry that project's `workspace_id`: `session_start`, `session_end`, `first_prompt_sent`, `first_job_completed`, `skill_used`, `skill_published`, `memory_injection`, `agent_outcome`, `task_classified`, `task_outcome_signal`, `tool_chain_outcome`, `core_failure`, `error_fingerprint`, `plan_revision`, `sql_quality` and `tool_call`. +- `session_end` counts the workspace skills, memory blocks and integration calls the session used. `memory_injection` counts the workspace's blocks. `task_outcome_signal` says whether workspace context was used. +- A call to a workspace integration through the engine is reported as a `tool_call` with `tool_type: "mcp"` and `via_workspace: true`. As for other tools, there are no arguments or outputs, and an error message is masked and capped at 500 characters. +- An `endpoint` value is a route label with every id and path segment replaced by `:id`, for example `POST /datamate-project-bindings/bind`. It never contains an id, a file path or a query string. + ### Notes on the review events - `degraded` is a fidelity flag, not a warehouse flag. It is set when a review found no reviewable diff --git a/packages/opencode/src/altimate/plugin/altimate.ts b/packages/opencode/src/altimate/plugin/altimate.ts index b9e0fbe4fc..baf989aa1d 100644 --- a/packages/opencode/src/altimate/plugin/altimate.ts +++ b/packages/opencode/src/altimate/plugin/altimate.ts @@ -12,6 +12,8 @@ import { Config } from "@/config/config" import { Flag } from "@/flag/flag" import { InstallationVersion } from "@opencode-ai/core/installation/version" import { Log } from "@/altimate/util/log" +// altimate_change — reads the config telemetry opt-out without an Instance +import { TelemetryConfig } from "../telemetry/config-state" /** * Why a failure reason is attached at the rejection site rather than inferred from the message: @@ -75,7 +77,9 @@ const log = Log.create({ service: "altimate-plugin" }) // and `cli_version` are strings, treat `cli_version: "local"` as a valid dev // build, and treat the payload as untrusted (anyone can craft a URL). An absent // `machine_id` means "do not attribute" — it must never be aliased on. -export async function buildCliContext(machineIdPath?: string): Promise { +/** `directory`: the project the command acts on, whose config may opt out; the process's working + * directory when the command has none (`auth login`). */ +export async function buildCliContext(machineIdPath?: string, directory?: string): Promise { // altimate_change start — honour both telemetry opt-out gates, mirroring // telemetry/index.ts::doInit: // 1. ALTIMATE_TELEMETRY_DISABLED / OPENCODE_DISABLE_TELEMETRY env vars @@ -90,17 +94,25 @@ export async function buildCliContext(machineIdPath?: string): Promise { } catch (err) { // Config was unreadable — NOT the normal path. Server routes run inside // Instance.provide() (AsyncLocalStorage-propagated across awaits), so - // Config.get() resolves during an ordinary browser authorize(). The known - // exception is `altimate auth login `, which deliberately skips - // instance bootstrap (ProvidersLoginCommand `instance: (args) => !isAuthProviderUrl(args.target)`); - // on that path this fires. Fail CLOSED — omit the durable machine_id rather - // than transmit it for a user who may have opted out via config; a missed - // correlation beats leaking a stable identifier. Log so a low correlation - // rate is traceable here instead of being mistaken for a lost reply. - log.warn("cli_context: config unreadable, omitting machine_id (fail-closed)", { + // Config.get() resolves during an ordinary browser authorize(). It fires on + // the paths that skip instance bootstrap: `altimate auth login ` and the + // create-and-link hand-off of `altimate-code link`. Read `telemetry.disabled` + // from the config files directly: an opt-out there, or files that cannot be + // read (an opt-out may be behind them), omit the id. Only a config that + // readably leaves telemetry on defers to the telemetry module's own state + // (opted out by env, no sink, test runner). + const state = await TelemetryConfig.disabledState(directory) + if (state === "enabled") { + await Telemetry.init().catch(() => {}) + disabled = !Telemetry.isEnabled() + } else { + disabled = true + } + log.info("cli_context: config unavailable, read the opt-out from the files", { code: (err as NodeJS.ErrnoException)?.code, + state, + enabled: !disabled, }) - disabled = true } } let machineId = "" diff --git a/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts b/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts index 4fed174ee1..b249f7fe79 100644 --- a/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts +++ b/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts @@ -261,7 +261,13 @@ export async function OnboardingTelemetryPlugin(_input: PluginInput): Promise { return { type: "error_fingerprint", timestamp: Date.now(), diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index ff983b90f4..2b7af62c48 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -248,6 +248,8 @@ export namespace Telemetry { // altimate_change start — add os/arch/node_version for environment segmentation | { type: "session_start" + /** altimate_change — the turn's workspace, named by the emitter */ + workspace_id?: number timestamp: number session_id: string model_id: string @@ -262,12 +264,18 @@ export namespace Telemetry { // altimate_change end | { type: "session_end" + /** altimate_change — the turn's own workspace, set by the prompt loop's end-of-turn rows */ + workspace_id?: number timestamp: number session_id: string total_cost: number total_tokens: number tool_call_count: number duration_ms: number + // altimate_change — what of the bound workspace this session used (see noteWorkspaceUse) + workspace_skills_used?: number + workspace_memory_blocks?: number + workspace_integration_calls?: number } | { type: "generation" @@ -326,6 +334,10 @@ export namespace Telemetry { previous_tool: string | null input_signature?: string error?: string + // altimate_change — the call went to the bound workspace (a datamate MCP server) + via_workspace?: boolean + // altimate_change — the turn's workspace, named by the MCP path on a workspace call + workspace_id?: number } | { type: "native_call" @@ -523,6 +535,8 @@ export namespace Telemetry { } | { type: "agent_outcome" + /** altimate_change — the turn's own workspace, set by the prompt loop's end-of-turn rows */ + workspace_id?: number timestamp: number session_id: string agent: string @@ -548,6 +562,164 @@ export namespace Telemetry { recovered: boolean duration_ms: number } + // altimate_change start — workspace link lifecycle (shared event contract: one name per fact, + // the backend's workspace id on every row, fired on the API success response, a `client` + // property stamped on every envelope). The extension emits the same names for its own moments. + | { + type: "workspace_linked" + timestamp: number + session_id: string + workspace_id: number + how: "bind_existing" | "create_and_bind" | "rebind_remote" | "rebind_path" + } + | { + type: "workspace_created" + timestamp: number + session_id: string + workspace_id: number + how: "create_and_bind" | "create_unbound" + } + | { + type: "workspace_link_failed" + timestamp: number + session_id: string + how: "bind_existing" | "create_and_bind" | "create_unbound" | "rebind_remote" | "rebind_path" + reason: "auth" | "not_found" | "network" | "conflict" | "other" + workspace_id?: number + } + | { + type: "workspace_unlinked" + timestamp: number + session_id: string + workspace_id?: number + } + | { + type: "workspace_switched" + timestamp: number + session_id: string + to_workspace_id: number + from_workspace_id?: number + how: "rebind_remote" | "rebind_path" + } + | { + type: "workspace_auth_expired" + timestamp: number + session_id: string + /** Method and path with ids replaced by ":id", e.g. "POST /datamate-project-bindings/bind". */ + endpoint: string + status: number + workspace_id?: number + } + | { + type: "workspace_api_error" + timestamp: number + session_id: string + endpoint: string + /** `response`: the server answered 2xx with an empty, unparseable or oversized body. */ + status_class: "4xx" | "5xx" | "network" | "timeout" | "response" + status?: number + workspace_id?: number + } + | { + /** The post-link starter message was served (TUI or extension chat); what the workspace provides. */ + type: "starter_shown" + timestamp: number + session_id: string + workspace_id: number + /** Absent when the starter could not tell (unsynced, unreadable, timed out). */ + skills?: number + integrations?: number + knowledge: "off" | "all" | "selected" | "unknown" + memory_enabled?: boolean + } + | { + /** One selection from the TUI's workspace starter: a suggested prompt, open in browser, or done. */ + type: "starter_action_picked" + timestamp: number + session_id: string + workspace_id: number + action: "prompt" | "open" | "done" + /** 0-based position of the suggested prompt; only for `prompt`. The prompt's text is never sent. */ + prompt_index?: number + } + | { + /** A memory block was written back to the bound workspace's cloud memory. */ + type: "workspace_memory_contributed" + timestamp: number + session_id: string + workspace_id: number + block_count: number + } + | { + /** A kind's sync status flipped to error. */ + type: "workspace_sync_failed" + timestamp: number + session_id: string + workspace_id: number + kind: "skills" | "memory" + reason: "auth" | "not_found" | "network" | "other" + } + // Workspace engine (see workspace/engine-telemetry.ts) + | { + /** A session's engine attach settled at a new outcome. */ + type: "workspace_engine_attach" + timestamp: number + session_id: string + outcome: "attached" | "engine_missing" | "engine_too_old" | "connect_failed" | "disabled" | "unbound" + /** The workspace the boundary settled for; absent for unbound and disabled. */ + workspace_id?: number + available?: number + declared?: number + missing?: number + extensions?: number + engine_version?: string + found_version?: string + error_class?: "link_unreadable" | "probe_failed" | "not_connected" | "other" + } + | { + type: "workspace_engine_install_prompt" + timestamp: number + session_id: string + action: "shown" | "accepted" | "installed" | "install_failed" | "copied" | "declined" + reason: "engine-missing" | "engine-too-old" + workspace_id?: number + declared?: number + found_version?: string + surface?: "headless" | "dialog" | "bus" | "toast" + node_detected?: boolean + node_major?: number + } + | { + /** A session's precedence snapshot settled at a new state. */ + type: "workspace_engine_precedence" + timestamp: number + session_id: string + enabled: boolean + reason?: string + shadowed_types: number + shadowed_connections: number + extensions: number + escape_hatch: boolean + workspace_id?: number + } + | { + /** A datamate tool call refused because the engine is off for that workspace. */ + type: "workspace_engine_redirect" + timestamp: number + session_id: string + tool: string + operation: string + outcome: "refused" + workspace_id?: number + } + | { + type: "workspace_engine_error" + timestamp: number + session_id: string + stage: "attach" | "install" | "precedence" + message: string + } + // altimate_change end | { type: "mcp_server_census" timestamp: number @@ -578,12 +750,16 @@ export namespace Telemetry { } | { type: "memory_injection" + /** altimate_change — the turn's workspace, named by the emitter */ + workspace_id?: number timestamp: number session_id: string block_count: number total_chars: number budget: number scopes_used: string[] + /** Blocks that came from the bound workspace's cloud memory (overlay). */ + workspace_blocks?: number } | { // Sent before connecting, so an attempt that hangs or ends the process is still counted. @@ -648,11 +824,13 @@ export namespace Telemetry { } | { type: "skill_used" + /** altimate_change — the turn's workspace, named by the emitter */ + workspace_id?: number timestamp: number session_id: string message_id: string skill_name: string - skill_source: "builtin" | "global" | "project" + skill_source: "builtin" | "global" | "project" | "workspace" duration_ms: number // altimate_change start — skill trigger classification for discovery analytics trigger: "user_command" | "llm_selected" | "auto_suggested" | "unknown" @@ -774,11 +952,15 @@ export namespace Telemetry { file_count: number // "serve": published from the IDE extension over the serve route source: "cli" | "tui" | "serve" + // altimate_change — the workspace the skill was attached to + workspace_id?: number } // altimate_change end // altimate_change start — plan refinement telemetry event | { type: "plan_revision" + /** altimate_change — the turn's workspace, named by the emitter */ + workspace_id?: number timestamp: number session_id: string revision_number: number @@ -807,6 +989,8 @@ export namespace Telemetry { // altimate_change end | { type: "core_failure" + /** altimate_change — the turn's workspace, named by the emitter */ + workspace_id?: number timestamp: number session_id: string tool_name: string @@ -861,6 +1045,8 @@ export namespace Telemetry { // altimate_change start — sql quality telemetry for issue prevention metrics | { type: "sql_quality" + /** altimate_change — the turn's workspace, named by the emitter */ + workspace_id?: number timestamp: number session_id: string tool_name: string @@ -875,6 +1061,8 @@ export namespace Telemetry { // implicit quality signal for task outcome intelligence | { type: "task_outcome_signal" + /** altimate_change — the turn's own workspace, set by the prompt loop's end-of-turn rows */ + workspace_id?: number timestamp: number session_id: string /** Behavioral signal derived from session outcome patterns */ @@ -887,10 +1075,14 @@ export namespace Telemetry { duration_ms: number /** Last tool category the agent used (or "none") */ last_tool_category: string + // altimate_change — true when the session had used workspace skills, memory or integrations + workspace_context_used?: boolean } // task intent classification for understanding DE problem distribution | { type: "task_classified" + /** altimate_change — the turn's workspace, named by the emitter */ + workspace_id?: number timestamp: number session_id: string /** Classified intent category */ @@ -950,6 +1142,8 @@ export namespace Telemetry { // error pattern fingerprint — hashed error grouping with recovery data | { type: "error_fingerprint" + /** altimate_change — the turn's own workspace, set by the prompt loop's end-of-turn rows */ + workspace_id?: number timestamp: number session_id: string /** SHA256 hash of normalized (masked) error message for grouping */ @@ -970,6 +1164,8 @@ export namespace Telemetry { // tool chain effectiveness — aggregated tool sequence + outcome at session end | { type: "tool_chain_outcome" + /** altimate_change — the turn's own workspace, set by the prompt loop's end-of-turn rows */ + workspace_id?: number timestamp: number session_id: string /** JSON-encoded ordered tool names (capped at 50) */ @@ -1175,6 +1371,8 @@ export namespace Telemetry { | { /** derived — see activation_job_selected */ type: "first_job_completed" + /** altimate_change — the turn's workspace, named by the emitter */ + workspace_id?: number timestamp: number session_id: string job: "sample_duck_db" | "breaks_downstream" | "sql_review" | "cost" | "something_else" @@ -1197,6 +1395,8 @@ export namespace Telemetry { } | { type: "first_prompt_sent" + /** altimate_change — the turn's workspace, named by the emitter */ + workspace_id?: number timestamp: number session_id: string } @@ -1922,6 +2122,13 @@ export namespace Telemetry { let machineId = "" let sessionId = "" let projectId = "" + // altimate_change start — the current turn token per session: `setContext` issues one, `endSession` + // ends only its own, so a cancelled turn unwinding late cannot end the replacement's counters. + // Tokens come from one process-wide counter, so a token is never reused. The map holds sessions + // mid-turn; an entry goes at the turn's end. + const sessionTurns = new Map() + let nextTurn = 0 + // altimate_change end let appInsights: AppInsightsConfig | undefined let droppedEvents = 0 let initPromise: Promise | undefined @@ -2011,6 +2218,9 @@ export namespace Telemetry { const properties: Record = { cli_version: InstallationVersion, + // altimate_change — the emitting client, per the shared event contract; `source` below + // still says how this CLI was launched (cli, tui, serve, datamates). + client: "altimate-code", source: clientSource, project_id: fields.project_id ?? projectId, ...(mid && { machine_id: mid }), @@ -2028,7 +2238,9 @@ export namespace Telemetry { for (const [k, v] of Object.entries(fields)) { if (k === "session_id" || k === "project_id" || k === "_retried" || k === "_operation_id") continue - if (typeof v === "number") { + // altimate_change — an id is a dimension, not a quantity: numeric *_id fields (workspace ids) + // go to properties as strings so the dashboards join on customDimensions like every other key. + if (typeof v === "number" && !k.endsWith("_id")) { measurements[k] = v } else if (v !== undefined && v !== null) { properties[k] = typeof v === "object" ? JSON.stringify(v) : String(v) @@ -2206,6 +2418,50 @@ export namespace Telemetry { } } + // altimate_change start — per-session use of the bound workspace. The skill, memory and tool + // emit sites note what they used; session_end reports the counts and task_outcome_signal + // whether any workspace context was in play. Keyed by session so a serve process with several + // sessions keeps them apart; cleared when the session ends. + export type WorkspaceUse = { skills: number; memory: number; integrations: number } + /** Keyed by session and turn, so a cancelled turn unwinding late and its replacement never share + * counts; a site that carries no turn counts toward the session's current one. */ + const workspaceUse = new Map() + const useKey = (sessionId: string, turn?: number) => `${sessionId}#${turn ?? sessionTurns.get(sessionId) ?? 0}` + /** `add` accumulates; `max` records the largest value seen in the turn (memory is injected on every + * step of a turn, so a sum would multiply the blocks by the step count). */ + export function noteWorkspaceUse( + sessionId: string, + kind: keyof WorkspaceUse, + n = 1, + mode: "add" | "max" = "add", + /** The turn the use belongs to (from `setContext`): a use landing after its turn was replaced + * does not count toward the replacement. */ + turn?: number, + ) { + if (!sessionId || n <= 0) return + if (turn !== undefined && sessionTurns.get(sessionId) !== turn) return + const key = useKey(sessionId, turn) + const use = workspaceUse.get(key) ?? { skills: 0, memory: 0, integrations: 0 } + use[kind] = mode === "max" ? Math.max(use[kind], n) : use[kind] + n + workspaceUse.set(key, use) + } + /** The counts of a turn (the session's current one when no turn is given). */ + export function workspaceUseOf(sessionId: string, turn?: number): WorkspaceUse { + return { skills: 0, memory: 0, integrations: 0, ...workspaceUse.get(useKey(sessionId, turn)) } + } + export function clearWorkspaceUse(sessionId: string, turn?: number) { + workspaceUse.delete(useKey(sessionId, turn)) + } + /** A session's turn ended: its use counts are read out and its `session_end` tracked by then. With + * `turn` (from `setContext`), a turn that is no longer the session's current one ends only its own + * counts: the replacement's are the replacement's. */ + export function endSession(sessionId: string, turn?: number) { + workspaceUse.delete(useKey(sessionId, turn)) + if (turn !== undefined && sessionTurns.get(sessionId) !== turn) return + sessionTurns.delete(sessionId) + } + // altimate_change end + // altimate_change start — re-resolve the signed-in user after a sign-in or sign-out in this // process (the browser login flow saves the credential while telemetry is already running). function resolveIdentity(opts: { refresh?: boolean } = {}): Promise { @@ -2311,12 +2567,19 @@ export namespace Telemetry { } // altimate_change end - export function setContext(opts: { sessionId: string; projectId: string }) { + /** Returns the turn generation for the session; pass it to `endSession` so only this turn ends it. */ + export function setContext(opts: { sessionId: string; projectId: string }): number { sessionId = opts.sessionId projectId = opts.projectId + // altimate_change start — a turn token for the session's counters and its end + if (!opts.sessionId) return 0 + const turn = ++nextTurn + sessionTurns.set(opts.sessionId, turn) + return turn + // altimate_change end } - export function getContext() { + export function getContext(): { sessionId: string; projectId: string } { return { sessionId, projectId } } @@ -2485,6 +2748,10 @@ export namespace Telemetry { const operationId = typeof eventSession === "string" ? TraceContext.activeTraceId(eventSession) : undefined if (operationId) (event as any)._operation_id = operationId // altimate_change end + // altimate_change — no ambient workspace stamping: a row names its workspace itself (the prompt + // loop's rows, tool rows and memory rows carry the turn's; workspace rows carry their own) or it + // carries none. A process-wide or per-session fallback kept attributing cancelled turns' rows to + // their replacements. buffer.push(event) if (buffer.length > MAX_BUFFER_SIZE) { buffer.shift() diff --git a/packages/opencode/src/altimate/tool-zod-compat.ts b/packages/opencode/src/altimate/tool-zod-compat.ts index 25938d1efd..ccab23a3cd 100644 --- a/packages/opencode/src/altimate/tool-zod-compat.ts +++ b/packages/opencode/src/altimate/tool-zod-compat.ts @@ -30,6 +30,10 @@ export type Metadata = { [key: string]: any; error?: string } export interface LegacyContext { sessionID: SessionID messageID: MessageID + // altimate_change start — the calling turn's workspace and token, as on Tool.Context + workspaceId?: number + turn?: number + // altimate_change end agent: string abort: AbortSignal callID?: string @@ -228,6 +232,10 @@ function legacyDefToDef(legacy: LegacyToolDef): DefWithoutID { const legacyCtx: LegacyContext = { sessionID: ctx.sessionID, messageID: ctx.messageID, + // altimate_change start — carried through for the rows a legacy tool emits + workspaceId: ctx.workspaceId, + turn: ctx.turn, + // altimate_change end agent: ctx.agent, abort: ctx.abort, callID: ctx.callID, diff --git a/packages/opencode/src/altimate/tools/datamate.ts b/packages/opencode/src/altimate/tools/datamate.ts index e7f2da3827..b2ea4c4d8d 100644 --- a/packages/opencode/src/altimate/tools/datamate.ts +++ b/packages/opencode/src/altimate/tools/datamate.ts @@ -79,14 +79,14 @@ export const DatamateManagerTool = Tool.define("datamate_manager", { .optional() .describe("Server name to remove (for 'remove'). Use 'list-config' or 'status' to find names."), }), - async execute(args): Promise<{ title: string; metadata: Record; output: string }> { + async execute(args, ctx): Promise<{ title: string; metadata: Record; output: string }> { // altimate_change start — a linked project gets its integrations from the // workspace's own engine only. The tool is kept out of the model's catalog // there (datamate-manager-gate.ts); this covers callers that run it // directly. Every operation is refused before anything is looked up or // written, so the refusal does not depend on the API being reachable. const workspace = await disablingWorkspace() - if (workspace) return refusal(args.operation, workspace) + if (workspace) return refusal(args.operation, workspace, ctx.sessionID) // altimate_change end if (args.operation !== "status" && args.operation !== "list-config") { const configured = await AltimateApi.isConfigured() @@ -106,9 +106,9 @@ export const DatamateManagerTool = Tool.define("datamate_manager", { case "list-integrations": return handleListIntegrations() case "add": - return handleAdd(args) + return handleAdd(args, "add", ctx.sessionID) case "create": - return handleCreate(args) + return handleCreate(args, ctx.sessionID) case "edit": return handleEdit(args) case "delete": @@ -116,7 +116,7 @@ export const DatamateManagerTool = Tool.define("datamate_manager", { case "status": return handleStatus() case "remove": - return handleRemove(args) + return handleRemove(args, ctx.sessionID) case "list-config": return handleListConfig() } @@ -250,6 +250,8 @@ function mergeRefreshedEntry( async function handleAdd( args: { datamate_id?: string; name?: string; scope?: "project" | "global" }, operation = "add", + /** The calling session, for the refusal's telemetry row. */ + sessionID?: string, ) { if (!args.datamate_id) { return { @@ -315,7 +317,7 @@ async function handleAdd( // it, as a boundary starts the engine's, or it could replace the engine's. using lock = await holdDirectoryLock() const off = await disablingWorkspace() - if (off) return refusal(operation, off) + if (off) return refusal(operation, off, sessionID) // altimate_change end if (transport !== null) { @@ -452,7 +454,7 @@ async function handleAdd( else await removeMcpFromConfig(serverName, configPath) again.release() if (prior && prior.enabled !== false) await MCP.add(serverName, prior).catch(() => {}) - return refusal(operation, linked) + return refusal(operation, linked, sessionID) } } // altimate_change end @@ -492,14 +494,17 @@ async function handleAdd( } } -async function handleCreate(args: { - name?: string - description?: string - integration_ids?: string[] - memory_enabled?: boolean - privacy?: string - scope?: "project" | "global" -}) { +async function handleCreate( + args: { + name?: string + description?: string + integration_ids?: string[] + memory_enabled?: boolean + privacy?: string + scope?: "project" | "global" + }, + sessionID?: string, +) { if (!args.name) { return { title: "Datamate create: FAILED", @@ -514,7 +519,7 @@ async function handleCreate(args: { // altimate_change start — checked again before the datamate is created: a // boundary may have linked the project since the check in `execute`. const off = await disablingWorkspace() - if (off) return refusal("create", off) + if (off) return refusal("create", off, sessionID) // The account the datamate is created in, kept for deleting it again: ids // are per tenant, and the credentials can change while the request runs. const creds = await AltimateApi.getCredentials() @@ -535,6 +540,7 @@ async function handleCreate(args: { const added = await handleAdd( { datamate_id: created.id, name: `datamate-${slugify(args.name)}`, scope: args.scope }, "create", + sessionID, ) if (!("managedBy" in added.metadata)) return added const rolledBack = await AltimateApi.deleteDatamate(created.id, creds).then( @@ -691,7 +697,7 @@ async function handleStatus() { } } -async function handleRemove(args: { server_name?: string; scope?: "project" | "global" }) { +async function handleRemove(args: { server_name?: string; scope?: "project" | "global" }, sessionID?: string) { if (!args.server_name) { return { title: "Datamate remove: FAILED", @@ -706,7 +712,7 @@ async function handleRemove(args: { server_name?: string; scope?: "project" | "g // check at the top of `execute` must not have its engine's key removed. using _lock = await holdDirectoryLock() const off = await disablingWorkspace() - if (off) return refusal("remove", off) + if (off) return refusal("remove", off, sessionID) // altimate_change end // Fully remove from runtime state (disconnect + purge from MCP list) // altimate_change start — MCP.remove (was disconnect): delete the status entry + publish diff --git a/packages/opencode/src/altimate/tools/training-import.ts b/packages/opencode/src/altimate/tools/training-import.ts index 5cac31512b..39660c9a63 100644 --- a/packages/opencode/src/altimate/tools/training-import.ts +++ b/packages/opencode/src/altimate/tools/training-import.ts @@ -116,6 +116,7 @@ export const TrainingImportTool = Tool.define("training_import", { scope: args.scope, content: section.content.slice(0, 1800), // Enforce max content length source: args.file_path, + sessionID: ctx.sessionID, }) imported++ results.push(` + ${section.name}`) diff --git a/packages/opencode/src/altimate/tools/training-save.ts b/packages/opencode/src/altimate/tools/training-save.ts index dd0880005c..8e83b63c9d 100644 --- a/packages/opencode/src/altimate/tools/training-save.ts +++ b/packages/opencode/src/altimate/tools/training-save.ts @@ -98,6 +98,7 @@ export const TrainingSaveTool = Tool.define("training_save", { content: args.content, source: args.source, citations: args.citations, + sessionID: ctx.sessionID, }) // Build response with context diff --git a/packages/opencode/src/altimate/tools/warehouse-add.ts b/packages/opencode/src/altimate/tools/warehouse-add.ts index 54adc86657..fb4803240a 100644 --- a/packages/opencode/src/altimate/tools/warehouse-add.ts +++ b/packages/opencode/src/altimate/tools/warehouse-add.ts @@ -120,7 +120,8 @@ IMPORTANT: For private key file paths, always use "private_key_path" (not "priva Telemetry.track({ type: "core_failure", timestamp: Date.now(), - session_id: Telemetry.getContext().sessionId || "unknown-session", + session_id: ctx.sessionID, + workspace_id: ctx.workspaceId, tool_name: "warehouse_add", tool_category: "warehouse", error_class: "internal", diff --git a/packages/opencode/src/altimate/training/store.ts b/packages/opencode/src/altimate/training/store.ts index 8597465d85..599ffdf43c 100644 --- a/packages/opencode/src/altimate/training/store.ts +++ b/packages/opencode/src/altimate/training/store.ts @@ -34,6 +34,8 @@ export namespace TrainingStore { content: string source?: string citations?: MemoryBlock["citations"] + /** The writing session, for the workspace write-back's row. */ + sessionID?: string }): Promise<{ entry: TrainingEntry; duplicates: MemoryBlock[] }> { const id = trainingId(input.kind, input.name) const existing = await MemoryStore.read(input.scope, id) @@ -48,15 +50,18 @@ export namespace TrainingStore { const enriched = embedTrainingMeta(input.content, meta) - const { duplicates } = await MemoryStore.write({ - id, - scope: input.scope, - tags: trainingTags(input.kind), - created: existing?.created ?? now, - updated: now, - citations: input.citations, - content: enriched, - }) + const { duplicates } = await MemoryStore.write( + { + id, + scope: input.scope, + tags: trainingTags(input.kind), + created: existing?.created ?? now, + updated: now, + citations: input.citations, + content: enriched, + }, + { sessionID: input.sessionID }, + ) return { entry: { diff --git a/packages/opencode/src/altimate/workspace/api-client.ts b/packages/opencode/src/altimate/workspace/api-client.ts index 15df318ff3..96ca3f0a7d 100644 --- a/packages/opencode/src/altimate/workspace/api-client.ts +++ b/packages/opencode/src/altimate/workspace/api-client.ts @@ -11,6 +11,7 @@ // bodies come out as `{"detail": }` — we parse the object form // for 409/412 and surface it as a typed error rather than a bare status code. import { AltimateApi } from "@/altimate/api/client" +import { Telemetry } from "@/altimate/telemetry" const REQUEST_TIMEOUT_MS = 15_000 @@ -153,6 +154,9 @@ export class WorkspaceApiError extends Error { constructor( msg: string, public readonly status?: number, + /** Where it failed: on the wire before any answer, or in an answer the server gave (its + * status, when known, is `status`). A statusless error is transport unless said otherwise. */ + public readonly failure: "transport" | "response" = status === undefined ? "transport" : "response", ) { super(msg) this.name = "WorkspaceApiError" @@ -173,6 +177,147 @@ async function creds(): Promise<{ url: string; instance: string; apiKey: string return { url: c.altimateUrl, instance: c.altimateInstanceName, apiKey: c.altimateApiKey } } +// altimate_change start — workspace telemetry helpers. Every failed workspace call reports +// `workspace_api_error` (or `workspace_auth_expired` on 401) with the endpoint masked of ids. A +// status the caller declares expected (a lookup's 404 is "not linked") is an answer, not a failure, +// and reports nothing; the link lifecycle functions below report linked / created / switched / unlinked on the +// success response and `workspace_link_failed` with a reason class otherwise. +/** Endpoint labels come from a ROUTE TABLE, never from how a segment is spelled: an identifier, + * a memory id or a file path equal to a route word must not reach telemetry. Each router the + * workspace clients call lists the shapes of its subpaths as regular expressions with the label + * they map to; a subpath that matches no shape, or a router not listed, is labelled with every + * segment masked. Identifiers and file paths therefore cannot leak whatever they contain; an + * unlisted new route only costs detail until it is added here. */ +const ROUTES: Record> = { + "/datamate-project-bindings": [ + [/^\/?$/, "/"], + [/^\/bind$/, "/bind"], + [/^\/by-remote$/, "/by-remote"], + [/^\/by-path$/, "/by-path"], + ], + "/datamates": [[/^\/?$/, "/"]], + "/skills": [ + [/^\/[^/]+$/, "/:id"], + [/^\/[^/]+\/datamates$/, "/:id/datamates"], + [/^\/[^/]+\/files(\/.*)?$/, "/:id/files/:path"], + ], + "/datamates/memory": [ + [/^\/?$/, "/"], + [/^\/list$/, "/list"], + [/^\/[^/]+$/, "/:id"], + ], +} + +/** "METHOD /base/path" with every dynamic segment replaced from the route table above. */ +export function endpointOf(method: string, basePath: string, subpath: string): string { + const shapes = ROUTES[basePath] + const maskAll = (p: string) => + p + .split("/") + .map((seg) => (seg === "" ? seg : ":id")) + .join("/") + if (!shapes) return `${method} ${maskAll(basePath)}${maskAll(subpath)}` + const query = subpath.split("?")[0] + for (const [shape, label] of shapes) if (shape.test(query)) return `${method} ${basePath}${label}` + return `${method} ${basePath}${maskAll(query)}` +} + +function workspaceIdIn(body: unknown): number | undefined { + if (!body || typeof body !== "object") return undefined + const b = body as Record + const id = b.datamate_id ?? b.target_datamate_id + return typeof id === "number" && Number.isInteger(id) && id > 0 ? id : undefined +} + +function reportApiErrorRow(input: { + endpoint: string + status?: number + kind: "4xx" | "5xx" | "network" | "timeout" | "response" + workspaceId?: number +}) { + const session_id = Telemetry.getContext().sessionId + const ws = input.workspaceId !== undefined ? { workspace_id: input.workspaceId } : {} + // Only a 401 says the session expired. A 403 is a refusal (a teammate's private workspace), so + // it is an API error with its status. + if (input.status === 401) { + Telemetry.track({ + type: "workspace_auth_expired", + timestamp: Date.now(), + session_id, + endpoint: input.endpoint, + status: input.status, + ...ws, + }) + return + } + Telemetry.track({ + type: "workspace_api_error", + timestamp: Date.now(), + session_id, + endpoint: input.endpoint, + status_class: input.kind, + ...(input.status !== undefined ? { status: input.status } : {}), + ...ws, + }) +} + +/** The reason class of a failed link call, for `workspace_link_failed`. */ +export function linkFailureReason(err: unknown): "auth" | "not_found" | "network" | "conflict" | "other" { + if (err instanceof ForbiddenError) return "auth" + if (err instanceof NotFoundError) return "not_found" + if (err instanceof ConflictError || err instanceof PreconditionFailedError) return "conflict" + if (err instanceof WorkspaceApiError) { + // An answer whose body failed carries its status here rather than as a typed error: the same + // status classes as a readable answer. + if (err.status === 401 || err.status === 403) return "auth" + if (err.status === 404) return "not_found" + if (err.status === 409 || err.status === 412) return "conflict" + if (err.status === undefined && err.failure === "transport") return "network" // unreachable or timed out + } + return "other" +} + +type LinkHow = "bind_existing" | "create_and_bind" | "create_unbound" | "rebind_remote" | "rebind_path" + +async function linkAttempt(how: LinkHow, workspaceId: number | undefined, call: () => Promise): Promise { + try { + return await call() + } catch (err) { + Telemetry.track({ + type: "workspace_link_failed", + timestamp: Date.now(), + session_id: Telemetry.getContext().sessionId, + how, + reason: linkFailureReason(err), + ...(workspaceId !== undefined ? { workspace_id: workspaceId } : {}), + }) + throw err + } +} + +function trackSwitched(to: number, from: number | undefined, how: "rebind_remote" | "rebind_path") { + Telemetry.track({ + type: "workspace_switched", + timestamp: Date.now(), + session_id: Telemetry.getContext().sessionId, + to_workspace_id: to, + ...(from !== undefined ? { from_workspace_id: from } : {}), + how, + }) + trackLinked(to, how) +} + +function trackLinked(workspaceId: number, how: "bind_existing" | "create_and_bind" | "rebind_remote" | "rebind_path") { + Telemetry.track({ + type: "workspace_linked", + timestamp: Date.now(), + session_id: Telemetry.getContext().sessionId, + workspace_id: workspaceId, + how, + }) +} +// altimate_change end + async function req( method: string, subpath: string, @@ -215,6 +360,13 @@ async function req( * It still completes as the principal it captured, and the next call fails * at its own credential read. */ actAs?: { url: string; instance: string; apiKey: string } + // altimate_change start — telemetry: statuses that are answers for this caller + /** Non-2xx statuses this caller handles as an answer, not a failure (a lookup's 404 is "not + * linked"). They still throw the typed error, but report no `workspace_api_error`. */ + expectedStatuses?: readonly number[] + /** false for a call that is not a workspace call (`GET /users/me`): it reports no row at all. */ + reportErrors?: boolean + // altimate_change end } = {}, ): Promise { const timeoutMs = opts.timeoutMs ?? REQUEST_TIMEOUT_MS @@ -222,10 +374,16 @@ async function req( const qs = opts.query ? "?" + new URLSearchParams(opts.query).toString() : "" const basePath = opts.base ?? "/datamate-project-bindings" const target = `${url}${basePath}${subpath}${qs}` + const endpoint = endpointOf(method, basePath, subpath) + const bodyWorkspaceId = workspaceIdIn(opts.body) + const reportApiError = (input: Parameters[0]) => { + if (opts.reportErrors !== false) reportApiErrorRow(input) + } const controller = new AbortController() const timeout = setTimeout(() => controller.abort(), timeoutMs) let res: Response let text: string + let answered: number | undefined // the status once the server answered, for the catch below try { res = await fetch(target, { method, @@ -238,6 +396,7 @@ async function req( signal: controller.signal, ...(opts.body !== undefined ? { body: JSON.stringify(opts.body) } : {}), }) + answered = res.status // Keep the AbortController timeout ACTIVE while we read the response body. // ``fetch()`` resolves after headers arrive; a server can send headers then // stall the body stream indefinitely, so pulling the body inside the same @@ -267,11 +426,34 @@ async function req( // The abort fires equally when it kills the fetch OR the body read. (m8) const name = (err as { name?: string } | undefined)?.name if (name === "AbortError") { - throw new WorkspaceApiError( - `Request to ${target} timed out after ${Math.round(timeoutMs / 1000)}s`, - ) + // A timeout after the server answered (the body stalled) keeps the answer's status: + // a 500 whose body never arrives is still a 5xx row; a 2xx whose body never arrives + // stays a timeout, with the status attached. + reportApiError({ + endpoint, + ...(answered !== undefined ? { status: answered } : {}), + kind: answered === undefined ? "timeout" : answered >= 500 ? "5xx" : answered >= 400 ? "4xx" : "timeout", + workspaceId: bodyWorkspaceId, + }) + // The answer's status rides on the error too, so a link failure after an answer is not + // classified as a network failure. + throw new WorkspaceApiError(`Request to ${target} timed out after ${Math.round(timeoutMs / 1000)}s`, answered) + } + // The server answered and the body failed afterwards (readBounded's cap, a cut stream): + // classify by the status it answered with, so an oversized 500 is still a 5xx row; only a + // usable status with a bad body is "response". Before any answer it is transport. + if (answered !== undefined) { + reportApiError({ + endpoint, + status: answered, + kind: answered >= 500 ? "5xx" : answered >= 400 ? "4xx" : "response", + workspaceId: bodyWorkspaceId, + }) + } else { + reportApiError({ endpoint, kind: "network", workspaceId: bodyWorkspaceId }) } const msg = err instanceof Error ? err.message : String(err) + if (answered !== undefined) throw new WorkspaceApiError(`Response from ${target} failed: ${msg}`, answered) throw new WorkspaceApiError(`Cannot reach ${target}: ${msg}`) } finally { clearTimeout(timeout) @@ -285,6 +467,14 @@ async function req( } } const detail = (json as { detail?: unknown } | undefined)?.detail + if (!res.ok && !opts.expectedStatuses?.includes(res.status)) { + reportApiError({ + endpoint, + status: res.status, + kind: res.status >= 500 ? "5xx" : "4xx", + workspaceId: bodyWorkspaceId, + }) + } if (res.status === 404) throw new NotFoundError(typeof detail === "string" ? detail : "Not found") if (res.status === 403) throw new ForbiddenError(typeof detail === "string" ? detail : "Forbidden") if (res.status === 409) { @@ -317,10 +507,8 @@ async function req( // is treated as an empty body too — otherwise ``null as T`` reaches callers // and .foo throws in a way the typed switches can't classify. (m7 + CR) if (json == null && !opts.allowEmptyBody) { - throw new WorkspaceApiError( - `Empty ${res.status} body from ${target} — expected JSON payload`, - res.status, - ) + reportApiError({ endpoint, status: res.status, kind: "response", workspaceId: bodyWorkspaceId }) + throw new WorkspaceApiError(`Empty ${res.status} body from ${target} — expected JSON payload`, res.status) } return json as T } @@ -347,9 +535,7 @@ async function readBounded(res: Response, target: string): Promise { if (!res.body) { const whole = await res.text() if (Buffer.byteLength(whole, "utf8") > MAX_RESPONSE_BYTES) { - throw new WorkspaceApiError( - `Response from ${target} exceeded the ${MAX_RESPONSE_BYTES} byte limit`, - ) + throw new WorkspaceApiError(`Response from ${target} exceeded the ${MAX_RESPONSE_BYTES} byte limit`) } return whole } @@ -363,9 +549,7 @@ async function readBounded(res: Response, target: string): Promise { if (!value) continue total += value.byteLength if (total > MAX_RESPONSE_BYTES) { - throw new WorkspaceApiError( - `Response from ${target} exceeded the ${MAX_RESPONSE_BYTES} byte limit`, - ) + throw new WorkspaceApiError(`Response from ${target} exceeded the ${MAX_RESPONSE_BYTES} byte limit`) } chunks.push(value) } @@ -381,7 +565,11 @@ export namespace WorkspaceApi { /** Server-authoritative pre-check by git remote. Returns null on 404. */ export async function getBindingForRemote(remote: string, actAs?: ActAs): Promise { try { - return await req("GET", "/by-remote", { query: { repo_remote: remote }, ...(actAs ? { actAs } : {}) }) + return await req("GET", "/by-remote", { + query: { repo_remote: remote }, + expectedStatuses: [404], + ...(actAs ? { actAs } : {}), + }) } catch (err) { if (err instanceof NotFoundError) return null throw err @@ -392,7 +580,11 @@ export namespace WorkspaceApi { * without a git remote). Returns null on 404. */ export async function getBindingForPath(projectPath: string, actAs?: ActAs): Promise { try { - return await req("GET", "/by-path", { query: { project_path: projectPath }, ...(actAs ? { actAs } : {}) }) + return await req("GET", "/by-path", { + query: { project_path: projectPath }, + expectedStatuses: [404], + ...(actAs ? { actAs } : {}), + }) } catch (err) { if (err instanceof NotFoundError) return null throw err @@ -404,7 +596,10 @@ export namespace WorkspaceApi { * picks the right endpoint even if the current identifier's remote has * changed since the binding was created (M3). Both fields on the * identifier are optional but at least one must be present. */ - export async function getBindingForProject(id: ProjectIdentifier, actAs?: ActAs): Promise { + export async function getBindingForProject( + id: ProjectIdentifier, + actAs?: ActAs, + ): Promise { if (id.repoRemote) { const hit = await getBindingForRemote(id.repoRemote, actAs) if (hit) return { ...hit, matchedBy: "remote" } @@ -426,7 +621,9 @@ export namespace WorkspaceApi { * A local-only unlink is not possible: ``lookupBinding`` re-asks the server * whenever the cache misses, so a row dropped only on disk comes straight back * on the next resolve. */ - export async function unbindProject(id: ProjectIdentifier): Promise { + /** `knownDatamateId`: the workspace the caller knows it is unlinking, for telemetry; the + * endpoint identifies the binding by project, not by workspace, and answers nothing. */ + export async function unbindProject(id: ProjectIdentifier, knownDatamateId?: number): Promise { const query: Record = {} // Send exactly one identifier. The endpoint answers 409 when both are given // and they name different bindings, and preferring the remote matches how @@ -436,7 +633,13 @@ export namespace WorkspaceApi { else if (id.projectPath) query.project_path = id.projectPath else return false try { - await req("DELETE", "/", { query, allowEmptyBody: true }) + await req("DELETE", "/", { query, allowEmptyBody: true, expectedStatuses: [404] }) + Telemetry.track({ + type: "workspace_unlinked", + timestamp: Date.now(), + session_id: Telemetry.getContext().sessionId, + ...(knownDatamateId !== undefined ? { workspace_id: knownDatamateId } : {}), + }) return true } catch (err) { if (err instanceof NotFoundError) return false @@ -449,14 +652,28 @@ export namespace WorkspaceApi { identifier: ProjectIdentifier description?: string }): Promise { - return req("POST", "/", { - body: { - name: input.name, - repo_remote: input.identifier.repoRemote ?? null, - project_path: input.identifier.projectPath ?? null, - description: input.description ?? null, - }, - }) + const res = await linkAttempt("create_and_bind", undefined, () => + req("POST", "/", { + body: { + name: input.name, + repo_remote: input.identifier.repoRemote ?? null, + project_path: input.identifier.projectPath ?? null, + description: input.description ?? null, + }, + }), + ) + const id = res?.binding?.datamate_id ?? res?.datamate?.id + if (typeof id === "number") { + Telemetry.track({ + type: "workspace_created", + timestamp: Date.now(), + session_id: Telemetry.getContext().sessionId, + workspace_id: id, + how: "create_and_bind", + }) + trackLinked(id, "create_and_bind") + } + return res } /** Create a workspace WITHOUT binding anything to it. @@ -503,30 +720,46 @@ export namespace WorkspaceApi { name: string description?: string }): Promise<{ id: number; name: string }> { - const data = await req<{ id: number }>("POST", "/", { - base: "/datamates", - body: { - name: input.name, - description: input.description ?? null, - integrations: [], - memory_enabled: true, - knowledge_engine_enabled: true, - privacy: "private", - }, + // The id check sits inside the reported attempt: a create whose answer is unusable is a + // failed link step, not a created workspace. + const id = await linkAttempt("create_unbound", undefined, async () => { + const data = await req<{ id: number }>("POST", "/", { + base: "/datamates", + body: { + name: input.name, + description: input.description ?? null, + integrations: [], + memory_enabled: true, + knowledge_engine_enabled: true, + privacy: "private", + }, + }) + // `typeof` FIRST, before any arithmetic. `Number()` coerces, so the + // previous `Number.isSafeInteger(Number(data?.id))` accepted `true` as 1, + // `"7"` as 7 and `[5]` as 5 — a malformed body would have rebound the + // project to whatever those coerced to (workspace 1, in the boolean case) + // instead of failing. The server's `CreateDatamateResponse` is `{id: int}` + // and FastAPI enforces it, so anything else here is a contract break worth + // refusing loudly rather than guessing at. + const id: unknown = (data as { id?: unknown } | null | undefined)?.id + if (typeof id !== "number" || !Number.isSafeInteger(id) || id <= 0) { + throw new WorkspaceApiError( + `Workspace was created but the server returned no usable id (${JSON.stringify(id) ?? "undefined"}).`, + undefined, + "response", + ) + } + return id + }) + // The create-then-rebind path for an already-linked project: the workspace exists + // from here; the rebind that follows reports the switch and the link. + Telemetry.track({ + type: "workspace_created", + timestamp: Date.now(), + session_id: Telemetry.getContext().sessionId, + workspace_id: id, + how: "create_unbound", }) - // `typeof` FIRST, before any arithmetic. `Number()` coerces, so the - // previous `Number.isSafeInteger(Number(data?.id))` accepted `true` as 1, - // `"7"` as 7 and `[5]` as 5 — a malformed body would have rebound the - // project to whatever those coerced to (workspace 1, in the boolean case) - // instead of failing. The server's `CreateDatamateResponse` is `{id: int}` - // and FastAPI enforces it, so anything else here is a contract break worth - // refusing loudly rather than guessing at. - const id: unknown = (data as { id?: unknown } | null | undefined)?.id - if (typeof id !== "number" || !Number.isSafeInteger(id) || id <= 0) { - throw new WorkspaceApiError( - `Workspace was created but the server returned no usable id (${JSON.stringify(id) ?? "undefined"}).`, - ) - } return { id, name: input.name } } @@ -540,14 +773,18 @@ export namespace WorkspaceApi { identifier: ProjectIdentifier, actAs?: ActAs, ): Promise { - return req("POST", "/bind", { - body: { - datamate_id: datamateId, - repo_remote: identifier.repoRemote ?? null, - project_path: identifier.projectPath ?? null, - }, - ...(actAs ? { actAs } : {}), - }) + const res = await linkAttempt("bind_existing", datamateId, () => + req("POST", "/bind", { + body: { + datamate_id: datamateId, + repo_remote: identifier.repoRemote ?? null, + project_path: identifier.projectPath ?? null, + }, + ...(actAs ? { actAs } : {}), + }), + ) + trackLinked(res?.binding?.datamate_id ?? datamateId, "bind_existing") + return res } export async function rebindByRemote(input: { @@ -556,16 +793,20 @@ export namespace WorkspaceApi { expectedCurrentDatamateId?: number actAs?: ActAs }): Promise { - return req("PUT", "/by-remote", { - body: { - repo_remote: input.remote, - target_datamate_id: input.targetDatamateId, - ...(input.expectedCurrentDatamateId !== undefined - ? { expected_current_datamate_id: input.expectedCurrentDatamateId } - : {}), - }, - ...(input.actAs ? { actAs: input.actAs } : {}), - }) + const res = await linkAttempt("rebind_remote", input.targetDatamateId, () => + req("PUT", "/by-remote", { + body: { + repo_remote: input.remote, + target_datamate_id: input.targetDatamateId, + ...(input.expectedCurrentDatamateId !== undefined + ? { expected_current_datamate_id: input.expectedCurrentDatamateId } + : {}), + }, + ...(input.actAs ? { actAs: input.actAs } : {}), + }), + ) + trackSwitched(res?.binding?.datamate_id ?? input.targetDatamateId, input.expectedCurrentDatamateId, "rebind_remote") + return res } /** Path-identified rebind — symmetric to ``rebindByRemote`` for projects @@ -576,16 +817,20 @@ export namespace WorkspaceApi { expectedCurrentDatamateId?: number actAs?: ActAs }): Promise { - return req("PUT", "/by-path", { - body: { - project_path: input.projectPath, - target_datamate_id: input.targetDatamateId, - ...(input.expectedCurrentDatamateId !== undefined - ? { expected_current_datamate_id: input.expectedCurrentDatamateId } - : {}), - }, - ...(input.actAs ? { actAs: input.actAs } : {}), - }) + const res = await linkAttempt("rebind_path", input.targetDatamateId, () => + req("PUT", "/by-path", { + body: { + project_path: input.projectPath, + target_datamate_id: input.targetDatamateId, + ...(input.expectedCurrentDatamateId !== undefined + ? { expected_current_datamate_id: input.expectedCurrentDatamateId } + : {}), + }, + ...(input.actAs ? { actAs: input.actAs } : {}), + }), + ) + trackSwitched(res?.binding?.datamate_id ?? input.targetDatamateId, input.expectedCurrentDatamateId, "rebind_path") + return res } /** Populates the "link to existing workspace" picker. Reuses the existing @@ -619,11 +864,7 @@ export namespace WorkspaceApi { // ``datamates`` or ``data`` value (object / string / null) would // otherwise slip through and throw on ``.map`` below, taking the // picker down before it renders. (cubic round 4.) - rows = Array.isArray(body.datamates) - ? body.datamates - : Array.isArray(body.data) - ? body.data - : [] + rows = Array.isArray(body.datamates) ? body.datamates : Array.isArray(body.data) ? body.data : [] } else { rows = [] } @@ -648,7 +889,12 @@ export namespace WorkspaceApi { * client must compare ownership — skill attachment requires the caller to * OWN the workspace, and the credentials carry no user id of their own. */ export async function whoami(actAs?: ActAs): Promise { - const me = await req<{ id?: unknown }>("GET", "/me", { base: "/users", ...(actAs ? { actAs } : {}) }) + // Not a workspace call: no workspace_api_error row for it. + const me = await req<{ id?: unknown }>("GET", "/me", { + base: "/users", + reportErrors: false, + ...(actAs ? { actAs } : {}), + }) const id = Number(me?.id) if (!Number.isInteger(id) || id <= 0) throw new WorkspaceApiError("The server did not say who this account is.") return id diff --git a/packages/opencode/src/altimate/workspace/browser-handoff.ts b/packages/opencode/src/altimate/workspace/browser-handoff.ts index 1f9cb6468d..65a0c7a8ec 100644 --- a/packages/opencode/src/altimate/workspace/browser-handoff.ts +++ b/packages/opencode/src/altimate/workspace/browser-handoff.ts @@ -556,7 +556,8 @@ export async function runHandoffWithOpener( // Import buildCliContext lazily so this module doesn't pull altimate.ts // into every consumer's import graph at load time. const { buildCliContext } = await import("../plugin/altimate") - const cliContext = await buildCliContext().catch((err) => { + // The linked project's config decides the opt-out, not the directory the command started in. + const cliContext = await buildCliContext(undefined, input.identifier.projectPath).catch((err) => { log.warn("buildCliContext failed; proceeding without", { err: String(err) }) return "" }) diff --git a/packages/opencode/src/altimate/workspace/datamate-manager-gate.ts b/packages/opencode/src/altimate/workspace/datamate-manager-gate.ts index 1fdfcf28ef..2c94bc2f7b 100644 --- a/packages/opencode/src/altimate/workspace/datamate-manager-gate.ts +++ b/packages/opencode/src/altimate/workspace/datamate-manager-gate.ts @@ -19,6 +19,7 @@ // key. Null for an unlinked project, an unreadable binding with no engine // running, workspaces disabled, `altimate serve`, and organisation-managed // config that owns the `datamate` key. +import { trackRedirect } from "./engine-telemetry" import { linkedWorkspaceLoaded, managedWorkspace, settledOutcome, settledWorkspace } from "./engine-overlay" import { installCommand } from "./engine-offer" import { log } from "./engine-seams" @@ -52,7 +53,8 @@ export async function hiddenToolIds(): Promise> { /** What `datamate_manager` answers in a linked project, whatever the operation. * Nothing is looked up or written before it. */ -export function refusal(operation: string, workspace: { id: string; name: string }) { +export function refusal(operation: string, workspace: { id: string; name: string }, sessionID?: string) { + trackRedirect("datamate", operation, Number(workspace.id), sessionID) // telemetry: the engine is off for this workspace return { title: `Datamate ${operation}: off in a project linked to a workspace`, metadata: { operation, managedBy: workspace.id }, diff --git a/packages/opencode/src/altimate/workspace/engine-offer.ts b/packages/opencode/src/altimate/workspace/engine-offer.ts index 4e282e1471..5d2b83983f 100644 --- a/packages/opencode/src/altimate/workspace/engine-offer.ts +++ b/packages/opencode/src/altimate/workspace/engine-offer.ts @@ -7,6 +7,7 @@ // the detail here from disk and PATH rather than from the overlay's memory. // Offer, never install on the flow's own account — `installEngine` only ever // runs from an explicit "Install now". +import { trackInstallPrompt } from "./engine-telemetry" import { execFile, type ChildProcess } from "node:child_process" import launch from "cross-spawn" import { Process } from "@/util/process" @@ -272,15 +273,61 @@ export async function installEngine(): Promise { return { ok: false, error: detail || `npm exited with code ${run.code ?? "unknown"}` } } +/** What the bus carries with the offer command: the offer's telemetry fields, so the surface that + * prints the offer in another process (an attached run) reports `shown` with them. No node probe + * rides along: it would sit on the dispatch path, ahead of the turn boundary. */ +export type OfferDetail = Pick + +export function encodeOfferDetail(offer: EngineOffer): string { + const detail: OfferDetail = { + reason: offer.reason, + workspaceId: offer.workspaceId, + declared: offer.declared, + found: offer.found, + } + return JSON.stringify(detail) +} + +/** The detail back from the bus, or undefined for anything that is not one. */ +export function decodeOfferDetail(raw: unknown): OfferDetail | undefined { + if (typeof raw !== "string") return undefined + try { + const v = JSON.parse(raw) as Partial> + const reason = v.reason === "engine-missing" || v.reason === "engine-too-old" ? v.reason : undefined + const workspaceId = typeof v.workspaceId === "string" ? v.workspaceId : undefined + const declared = typeof v.declared === "number" ? v.declared : v.declared === undefined ? undefined : NaN + const found = typeof v.found === "string" ? v.found : v.found === undefined ? undefined : "" + if (!reason || workspaceId === undefined || Number.isNaN(declared) || found === "") { + return undefined + } + return { reason, workspaceId, declared, found } + } catch { + return undefined + } +} + +/** An attached run printed the offer it read off the bus: that process is the surface that showed + * it, so it reports `shown` (surface `bus`) from the fields the event carried. False when the + * event carried none that decode, and nothing is reported. */ +export function trackPrintedOffer(raw: unknown, sessionID: string): boolean { + const detail = decodeOfferDetail(raw) + if (!detail) return false + trackInstallPrompt("shown", detail, { surface: "bus" }, sessionID) + return true +} + /** Ask the TUI to raise the offer. False when the bus is unavailable. The * session is carried so an attached headless run, which reads the same event - * stream, prints the offer raised for its own session only. */ -async function publishOffer(sessionID: string): Promise { - if (syncInternals.publishOffer) return syncInternals.publishOffer(sessionID) + * stream, prints the offer raised for its own session only; the offer's fields + * ride along so that run reports `shown` from them (the TUI dialog re-derives + * the offer and reports its own). */ +async function publishOffer(offer: EngineOffer, sessionID: string): Promise { + const detail = encodeOfferDetail(offer) + if (syncInternals.publishOffer) return syncInternals.publishOffer(sessionID, detail) try { await AppRuntime.runPromise( EventV2Bridge.Service.use((events) => - events.publish(TuiEvent.CommandExecute, { command: OFFER_COMMAND, sessionID }), + events.publish(TuiEvent.CommandExecute, { command: OFFER_COMMAND, sessionID, arguments: detail }), ), ) return true @@ -320,11 +367,20 @@ export function toolsNeed(declared: number | undefined): string { /** Offer via the dialog surface when there is one; otherwise print (headless) * or toast (bus unavailable). Exactly one of these happens. */ export async function offerOrNotify(offer: EngineOffer, toast: Toast, sessionID: string): Promise { + // Telemetry: the offer was shown, on which surface, and whether Node was there to install with. + const shown = (surface: "headless" | "dialog" | "bus" | "toast") => + void nodeMajor() + .catch(() => null) + .then((major) => trackInstallPrompt("shown", offer, { surface, nodeMajor: major }, sessionID)) if (isHeadless()) { printLine(describeOfferLine(offer)) + shown("headless") return } + // A dialog, whether taken here or over the bus, reports `shown` itself when it renders: the TUI + // may hold the offer back (its 7-day latch), so dispatch is not display. The toast reports here + // only when it was shown. if (offerInstall(offer)) return - if (await publishOffer(sessionID)) return - await notify(toast) + if (await publishOffer(offer, sessionID)) return + if (await notify(toast)) shown("toast") } diff --git a/packages/opencode/src/altimate/workspace/engine-overlay.ts b/packages/opencode/src/altimate/workspace/engine-overlay.ts index 074f940ae8..3d535c41ef 100644 --- a/packages/opencode/src/altimate/workspace/engine-overlay.ts +++ b/packages/opencode/src/altimate/workspace/engine-overlay.ts @@ -44,6 +44,7 @@ import { } from "./engine-seams" import { declaredBounded, fingerprint, notify, printLine, resolveBinding, versionOf, which } from "./engine-probes" import { OFFER_RECHECK_MS, OFFER_SKIP_TTL_MS, installCommand, offerOrNotify, type EngineOffer } from "./engine-offer" +import { attachKey, trackAttach, trackEngineError } from "./engine-telemetry" import { ENGINE_BINARY, INSTALL_HELPS, @@ -293,6 +294,7 @@ export async function overlay( log.warn("workspace engine overlay failed; leaving the MCP config as loaded", { err: String(err) }) state.current = null state.failedAt = now() + trackEngineError("attach", String(err)) } } @@ -335,6 +337,12 @@ export function linkedWorkspace(directory: string | null = currentDirectory()): return linked ? { id: linked.id, name: linked.name } : null } +/** The linked workspace's account-scoped identity (credential scope plus id), for telemetry's dedupe. */ +export function linkedWorkspaceKey(directory: string | null = currentDirectory()): string | undefined { + if (!directory) return undefined + return directories.get(directory)?.linked?.key +} + /** `linkedWorkspace` once the overlay has run for this instance. */ export async function linkedWorkspaceLoaded( directory: string | null = currentDirectory(), @@ -358,17 +366,33 @@ type SessionRecord = { /** The workspace the directory was linked to when the boundary settled this * outcome. */ linked?: { id: string; name: string } | null + /** The key of the last attach reported to telemetry (workspace and outcome); a transient record + * carries the previous one. */ + tracked?: string } const sessions = new Map() const declaredCache = new Map() /** Verdict signatures a headless process has already printed to stderr. */ const headlessPrinted = new Set() -function record(sessionID: string, outcome: Outcome): SessionRecord { +function record( + sessionID: string, + outcome: Outcome, + extra: { engineVersion?: string | null; transient?: boolean } = {}, +): SessionRecord { const previous = sessions.get(sessionID) + // telemetry: a settled outcome, when it or the workspace it settled for changed. A transient + // record (the retry sentinel) reports nothing and keeps the last reported key for the next + // comparison. + const workspaceId = Number(linkedWorkspace()?.id) || undefined + const tracked = extra.transient ? previous?.tracked : attachKey(outcome, linkedWorkspaceKey()) + if (!extra.transient) { + trackAttach(outcome, previous?.tracked, { ...extra, workspaceId, workspaceIdentity: linkedWorkspaceKey() }, sessionID) + } sessions.delete(sessionID) const next: SessionRecord = { outcome, + tracked, announced: previous?.announced, announcedAt: previous?.announcedAt, retried: previous?.retried, @@ -490,6 +514,7 @@ export async function atTurnStart(sessionID: string, body: () => Promise): await reconcile(sessionID, directory, state) } catch (err) { log.warn("workspace engine turn hook failed", { sessionID, err: String(err) }) + trackEngineError("attach", String(err), sessionID) } // Kept with the outcome, under the lock: the directory's link moves with any // later config load, and a turn's notice must name the workspace its own @@ -828,14 +853,20 @@ async function reconcile(sessionID: string, directory: string, state: DirectoryS let status = statusMap[DATAMATE_KEY] const session = sessions.get(sessionID) if (status?.status !== "connected" && !session?.retried) { - ;(session ?? record(sessionID, { kind: "connect-failed", error: "retrying" })).retried = true + ;(session ?? record(sessionID, { kind: "connect-failed", error: "retrying" }, { transient: true })).retried = true log.info("workspace engine not connected; retrying once for this session", { workspaceId: workspace.id, sessionID, status: status?.status, }) - await mcp().add(DATAMATE_KEY, overlayNow.entry) - status = (await mcp().status())[DATAMATE_KEY] + try { + await mcp().add(DATAMATE_KEY, overlayNow.entry) + status = (await mcp().status())[DATAMATE_KEY] + } catch (err) { + // A retry that throws settles as a failure for this session, rather than leaving the turn + // hook to swallow it with no outcome recorded. + status = { status: "failed", error: String(err) } + } } if (status?.status !== "connected") { const outcome: Outcome = { @@ -873,7 +904,7 @@ async function reconcile(sessionID: string, directory: string, state: DirectoryS ...(unfulfilled === undefined ? {} : { unfulfilled }), ...(declared?.extensions?.length ? { extensions: declared.extensions } : {}), } - const rec = record(sessionID, outcome) + const rec = record(sessionID, outcome, { engineVersion: overlayNow.version }) const snapshot: AttachSnapshot = { workspace: { id: workspace.id, name: workspace.name, key: workspace.key }, engineVersion: overlayNow.version, diff --git a/packages/opencode/src/altimate/workspace/engine-seams.ts b/packages/opencode/src/altimate/workspace/engine-seams.ts index 73d6609159..4ab0553a47 100644 --- a/packages/opencode/src/altimate/workspace/engine-seams.ts +++ b/packages/opencode/src/altimate/workspace/engine-seams.ts @@ -33,7 +33,7 @@ export const syncInternals: { printLine?: (line: string) => void /** Install-offer seams (see engine-offer.ts). */ offer?: (offer: EngineOffer) => boolean - publishOffer?: (sessionID: string) => Promise + publishOffer?: (sessionID: string, detail: string) => Promise runInstall?: ( argv: string[], timeoutMs: number, diff --git a/packages/opencode/src/altimate/workspace/engine-telemetry.ts b/packages/opencode/src/altimate/workspace/engine-telemetry.ts new file mode 100644 index 0000000000..75f34962d0 --- /dev/null +++ b/packages/opencode/src/altimate/workspace/engine-telemetry.ts @@ -0,0 +1,245 @@ +// altimate_change - new file +// +// Telemetry for the workspace engine, per the shared event contract: one name per fact, fired +// on the outcome rather than the attempt, the workspace id on the row where the code has it +// (every row of a bound session also carries it from the telemetry context). Each builder is +// pure so it can be pinned by tests; each `track*` wrapper never throws into its caller. +import { Telemetry } from "@/altimate/telemetry" +import type { Outcome } from "./engine-types" +import type { EngineOffer } from "./engine-offer" +import type { Precedence } from "./precedence" + +export type AttachOutcome = "attached" | "engine_missing" | "engine_too_old" | "connect_failed" | "disabled" | "unbound" +export type InstallPromptAction = "shown" | "accepted" | "installed" | "install_failed" | "copied" | "declined" +export type InstallPromptSurface = "headless" | "dialog" | "bus" | "toast" +export type EngineStage = "attach" | "install" | "precedence" + +const sessionId = () => Telemetry.getContext().sessionId + +function outcomeName(kind: Outcome["kind"]): AttachOutcome { + return kind.replace(/-/g, "_") as AttachOutcome +} + +/** The class of a connect failure, from the fixed messages the overlay produces. */ +export function connectErrorClass(error: string): "link_unreadable" | "probe_failed" | "not_connected" | "other" { + const e = error.toLowerCase() + if (e.includes("link could not be read")) return "link_unreadable" + if (e.includes("could not be checked")) return "probe_failed" + if (e.includes("not connected") || e.includes("did not connect") || e.includes("handshake")) return "not_connected" + return "other" +} + +/** What makes two settled outcomes the same attach for telemetry: the kind and its counts. */ +export function attachSignature(outcome: Outcome): string { + switch (outcome.kind) { + case "attached": + return `attached:${outcome.available}:${outcome.declared ?? ""}:${outcome.missing?.length ?? ""}:${outcome.extensions?.length ?? 0}` + case "engine-missing": + return `engine-missing:${outcome.declared ?? ""}` + case "engine-too-old": + return `engine-too-old:${outcome.found ?? ""}` + case "connect-failed": + return `connect-failed:${connectErrorClass(outcome.error)}` + default: + return outcome.kind + } +} + +export function attachEvent( + outcome: Outcome, + extra: { engineVersion?: string | null; workspaceId?: number } = {}, + sessionID?: string, +): Telemetry.Event { + // The boundary's own workspace, named on the row rather than left to the process context; an + // unbound or disabled outcome has none, and says so with an explicit undefined so the session's + // entry is not stamped onto it either (the envelope drops an undefined value). + const settledFor = outcome.kind !== "unbound" && outcome.kind !== "disabled" ? extra.workspaceId : undefined + const base = { + type: "workspace_engine_attach" as const, + timestamp: Date.now(), + session_id: sessionID ?? sessionId(), + workspace_id: settledFor !== undefined && Number.isInteger(settledFor) && settledFor > 0 ? settledFor : undefined, + } + switch (outcome.kind) { + case "attached": + return { + ...base, + outcome: "attached", + available: outcome.available, + ...(outcome.declared !== undefined ? { declared: outcome.declared } : {}), + ...(outcome.missing !== undefined ? { missing: outcome.missing.length } : {}), + ...(outcome.extensions?.length ? { extensions: outcome.extensions.length } : {}), + ...(extra.engineVersion ? { engine_version: extra.engineVersion } : {}), + } + case "engine-missing": + return { + ...base, + outcome: "engine_missing", + ...(outcome.declared !== undefined ? { declared: outcome.declared } : {}), + } + case "engine-too-old": + return { ...base, outcome: "engine_too_old", ...(outcome.found ? { found_version: outcome.found } : {}) } + case "connect-failed": + return { ...base, outcome: "connect_failed", error_class: connectErrorClass(outcome.error) } + default: + return { ...base, outcome: outcomeName(outcome.kind) } + } +} + +/** Track the attach when the session's outcome changed (not on every turn boundary). */ +/** What an attach row is deduped on: the directory's workspace identity (the credential scope plus the + * tenant-local id, as the overlay keys it, so the same id under another account is another workspace) + * and the outcome's signature; the same-shaped attach to another workspace after a relink is a change. */ +export function attachKey(outcome: Outcome, workspaceIdentity?: string): string { + return `${workspaceIdentity ?? ""}|${attachSignature(outcome)}` +} + +/** Track a session's attach when its key differs from the last one reported for the session. */ +export function trackAttach( + outcome: Outcome, + previousKey: string | undefined, + extra: { engineVersion?: string | null; workspaceId?: number; workspaceIdentity?: string } = {}, + sessionID?: string, +): boolean { + try { + const identity = + extra.workspaceIdentity ?? (extra.workspaceId !== undefined ? String(extra.workspaceId) : undefined) + if (previousKey !== undefined && previousKey === attachKey(outcome, identity)) return false + Telemetry.track(attachEvent(outcome, extra, sessionID)) + return true + } catch { + return false + } +} + +export function installPromptEvent( + action: InstallPromptAction, + offer: Pick, + opts: { surface?: InstallPromptSurface; nodeMajor?: number | null } = {}, + sessionID?: string, +): Telemetry.Event { + const id = Number(offer.workspaceId) + return { + type: "workspace_engine_install_prompt", + timestamp: Date.now(), + session_id: sessionID ?? sessionId(), + action, + reason: offer.reason, + ...(Number.isInteger(id) && id > 0 ? { workspace_id: id } : {}), + ...(offer.declared !== undefined ? { declared: offer.declared } : {}), + ...(offer.found ? { found_version: offer.found } : {}), + ...(opts.surface ? { surface: opts.surface } : {}), + ...(opts.nodeMajor !== undefined ? { node_detected: opts.nodeMajor !== null } : {}), + ...(typeof opts.nodeMajor === "number" ? { node_major: opts.nodeMajor } : {}), + } +} + +export function trackInstallPrompt( + action: InstallPromptAction, + offer: Pick, + opts: { surface?: InstallPromptSurface; nodeMajor?: number | null } = {}, + sessionID?: string, +): void { + try { + Telemetry.track(installPromptEvent(action, offer, opts, sessionID)) + } catch { + // never block the offer + } +} + +export function precedenceSignature( + p: Pick, +): string { + const shadowed = [...p.shadowed.entries()] + .map(([type, caps]) => `${type}=${caps.size}`) + .sort() + .join(",") + return `${p.workspaceId ?? ""}|${p.enabled}|${p.disabledReason ?? ""}|${shadowed}|${p.extensions?.length ?? 0}` +} + +export function precedenceEvent( + p: Pick, + opts: { escapeHatch: boolean; connections: number; types: number; sessionID?: string; workspaceId?: number }, +): Telemetry.Event { + // The snapshot's own workspace, else the directory's linked one passed by refresh(): a disabled + // snapshot (escape hatch, unattributed) names no workspace itself, and the row must not be left + // to the process context. + const own = Number(p.workspaceId) + const id = Number.isInteger(own) && own > 0 ? own : opts.workspaceId + return { + type: "workspace_engine_precedence", + timestamp: Date.now(), + session_id: opts.sessionID ?? sessionId(), + enabled: p.enabled, + ...(p.disabledReason ? { reason: p.disabledReason } : {}), + // Types the session can route through the engine (the inventory line's count), not every + // materialised type: a session whose rules deny the engine keys shadows nothing. + shadowed_types: opts.types, + shadowed_connections: opts.connections, + extensions: p.extensions?.length ?? 0, + escape_hatch: opts.escapeHatch, + // Present and undefined for an unbound or disabled snapshot with no linked workspace, as on the + // attach row: the row names no workspace, and nothing else may name one for it. + workspace_id: id !== undefined && Number.isInteger(id) && id > 0 ? id : undefined, + } +} + +const precedenceSeen = new Map() +/** Sessions remembered for the dedupe; the oldest is forgotten past this, and reports again on its next refresh. */ +const MAX_SEEN_SESSIONS = 256 + +/** Track a session's precedence snapshot when it changed since the last refresh. */ +export function trackPrecedence( + sessionID: string, + p: Pick, + opts: { escapeHatch: boolean; connections: number; types: number; identity?: string; workspaceId?: number }, +): boolean { + try { + // The local connections shadowed and the escape hatch are part of what is reported, so a change + // in either re-reports the same snapshot; the account-scoped identity (credential scope plus + // id) tells the same id under another account apart. + const signature = `${precedenceSignature(p)}|c=${opts.connections}|t=${opts.types}|e=${opts.escapeHatch}|w=${opts.identity ?? ""}` + if (precedenceSeen.get(sessionID) === signature) return false + precedenceSeen.set(sessionID, signature) + if (precedenceSeen.size > MAX_SEEN_SESSIONS) precedenceSeen.delete(precedenceSeen.keys().next().value!) + Telemetry.track(precedenceEvent(p, { ...opts, sessionID })) + return true + } catch { + return false + } +} + +export function resetEngineTelemetryForTests(): void { + precedenceSeen.clear() +} + +export function trackEngineError(stage: EngineStage, message: string, sessionID?: string): void { + try { + Telemetry.track({ + type: "workspace_engine_error", + timestamp: Date.now(), + session_id: sessionID ?? sessionId(), + stage, + message: Telemetry.maskString(message).slice(0, 200), + }) + } catch { + // never surface + } +} + +/** A datamate tool call refused because the engine is off for that workspace. */ +export function trackRedirect(tool: string, operation: string, workspaceId: number, sessionID?: string): void { + try { + Telemetry.track({ + type: "workspace_engine_redirect", + timestamp: Date.now(), + session_id: sessionID ?? sessionId(), + tool, + operation, + outcome: "refused", + ...(Number.isInteger(workspaceId) && workspaceId > 0 ? { workspace_id: workspaceId } : {}), + }) + } catch { + // never surface + } +} diff --git a/packages/opencode/src/altimate/workspace/manage.ts b/packages/opencode/src/altimate/workspace/manage.ts index da24513695..9ec8a3947c 100644 --- a/packages/opencode/src/altimate/workspace/manage.ts +++ b/packages/opencode/src/altimate/workspace/manage.ts @@ -411,6 +411,9 @@ export async function unlink(directory: string): Promise { // a blocking git call — is reached only for a project with no local row, which // is the case unlink exists to repair. let identifier: ProjectIdentifier + // The workspace being unlinked, for telemetry: the local row's id, else the one the + // server lookup below finds (the no-local-row recovery path). + let unlinkingId: number | undefined = was?.datamateId if (was?.repoRemote) identifier = { repoRemote: was.repoRemote } else if (was?.projectPath) identifier = { projectPath: was.projectPath } else { @@ -431,8 +434,9 @@ export async function unlink(directory: string): Promise { if (hit?.matchedBy === "path" && detected.projectPath) { identifier = { projectPath: detected.projectPath } } + unlinkingId ??= hit?.binding?.datamate_id } - const removedServerSide = await WorkspaceApi.unbindProject(identifier) + const removedServerSide = await WorkspaceApi.unbindProject(identifier, unlinkingId) // Only the row unlink started from. A relink that completed while the DELETE // was in flight recorded a new row, and removing that — then memoizing the diff --git a/packages/opencode/src/altimate/workspace/memory-sync.ts b/packages/opencode/src/altimate/workspace/memory-sync.ts index 63cff1314f..fe34c0fed0 100644 --- a/packages/opencode/src/altimate/workspace/memory-sync.ts +++ b/packages/opencode/src/altimate/workspace/memory-sync.ts @@ -16,6 +16,7 @@ // Nothing is written to disk: local files are the source of truth, and a cloud // record can have been edited elsewhere by a client that does not preserve this // CLI's metadata. +import { Telemetry } from "@/altimate/telemetry" import { createHash } from "node:crypto" import { Flag as CoreFlag } from "@opencode-ai/core/flag/flag" import { Flag } from "@/flag/flag" @@ -35,7 +36,8 @@ import { } from "./state" import { indexKey, readIndex, readIndexEntry, recordIndexEntry } from "./memory-index" import { projectIdentity, projectIdentityKey, sameProjectIdentity, type ProjectIdentity } from "./remote-identity" -import { WorkspaceApi } from "./api-client" +import { WorkspaceApi, WorkspaceApiError } from "./api-client" +import { redact } from "@/altimate/debug/report" // altimate_change — failure text in the sync record import { AltimateApi } from "@/altimate/api/client" import { LIST_LIMIT, @@ -307,6 +309,19 @@ async function memoryEnabled(binding: CachedBinding): Promise { return (await memoryStatus(binding)) === "enabled" } +/** A failed read says what failed, so the sync record can classify it. */ +type MemoryStatusOutcome = "enabled" | "disabled" | { status: "error"; error: string } + +// altimate_change start — what failed, for the persisted sync record: the HTTP status when the API +// answered one (so the reason class sees it whatever the detail says), redacted of secrets, URLs +// and the user's identity before it is written. +function failureText(err: unknown): string { + const status = err instanceof WorkspaceApiError ? err.status : undefined + const base = err instanceof Error ? `${err.name}: ${err.message}` : String(err) + return redact(status !== undefined ? `HTTP ${status} ${base}` : base) +} +// altimate_change end + /** Three-way, because a read that cannot reach the service must not be reported * as "this workspace has no memory" — that reads as success while destroying * whatever the session already had. */ @@ -319,7 +334,7 @@ async function memoryStatus( * inherit a positive written under a previous account. */ fresh?: boolean } = {}, -): Promise<"enabled" | "disabled" | "error"> { +): Promise { const cached = opts.fresh ? undefined : memoryEnabledCache.get(binding.datamateId) if (cached && Date.now() - cached.checkedAt < MEMORY_ENABLED_TTL_MS) return "enabled" try { @@ -335,7 +350,7 @@ async function memoryStatus( missingFromListWarned.add(binding.datamateId) log.warn("workspace missing from the workspace list; memory setting unknown", { workspace: binding.datamateId }) } - return "error" + return { status: "error", error: "workspace missing from the workspace list" } } missingFromListWarned.delete(binding.datamateId) if (match.memoryEnabled === undefined && !missingFieldWarned.has(binding.datamateId)) { @@ -362,7 +377,7 @@ async function memoryStatus( log.warn("could not confirm workspace memory setting", { err: String(err) }) // Not cached either way: a transient failure should neither disable the // mirror for a minute nor keep it enabled. - return "error" + return { status: "error", error: failureText(err) } } } @@ -379,11 +394,7 @@ function stripTrainingMeta(content: string): string { /** Fingerprint of everything about a block that reaches the store. Tags and * expiry are included because both are mirrored. */ function contentHash(block: MemoryBlock): string { - const payload = JSON.stringify([ - stripTrainingMeta(block.content), - [...block.tags].sort(), - block.expires ?? "", - ]) + const payload = JSON.stringify([stripTrainingMeta(block.content), [...block.tags].sort(), block.expires ?? ""]) // sha-256, not a 32-bit non-cryptographic hash. This value is the single // gate deciding whether a save is sent at all: on a collision ``push`` // returns "unchanged" and a real edit is silently never mirrored, with no @@ -702,7 +713,9 @@ async function reapArchivedBlocks( try { const gone = syncInternals.removeBlock ? await syncInternals.removeBlock(scope, blockId, directory ?? undefined, block.updated) - : await (await import("@/memory/store")).MemoryStore.remove(scope, blockId, directory ?? undefined, { + : await ( + await import("@/memory/store") + ).MemoryStore.remove(scope, blockId, directory ?? undefined, { expectUpdated: block.updated, }) if (gone) { @@ -721,6 +734,8 @@ async function push( binding: CachedBinding | null, known?: KnownRecords, directory?: string, + /** The session that wrote the block, captured before the queue; the context's when absent. */ + sessionID?: string, ): Promise { const key = indexKey({ scope: block.scope, @@ -770,9 +785,7 @@ async function push( // user recreated under an old id gets a fresh record instead of un-archiving // the old one (`MemoryApi.update` replaces metadata wholesale, which would // drop the archived marker). - const indexed = existing?.memoryId - ? view.records.find((r) => r.id === existing.memoryId) - : undefined + const indexed = existing?.memoryId ? view.records.find((r) => r.id === existing.memoryId) : undefined // A record the index names but a COMPLETE record set does not hold is gone — // deleted in the web app, by another client, or by a wipe. Absence used to // read as "still ours", so `match` stayed pointing at a dead id and every @@ -816,6 +829,7 @@ async function push( const others = copies.filter((r) => r.id !== match).map((r) => r.id) await archiveDuplicates(others, block, metadata, "archiving a second live record of one block") await recordIndexEntry(key, { memoryId: match, contentHash: hash, syncedAt: Date.now() }) + if (block.scope === "project" && binding) trackContributed(binding, sessionID) return "stored" } @@ -848,6 +862,9 @@ async function push( // leaves the block unindexed, and the next save finds the extras by identity. await archiveDuplicates(extras, block, metadata, "archiving an extra record produced by one create") await recordIndexEntry(key, { memoryId: primary, contentHash: hash, syncedAt: Date.now() }) + // The write-back is durable once indexed: reported here, so a failed archive above leaves the block + // unindexed and the retry that indexes it reports it, once. + if (block.scope === "project" && binding) trackContributed(binding, sessionID) return "stored" } @@ -866,6 +883,24 @@ async function archiveDuplicates( } } +// altimate_change start — a successful write-back of a project block is the workspace getting something +// back; a global block is the account's (the live mirror resolves a binding for it too, to consult the +// memory setting), so it is not reported. +function trackContributed(binding: CachedBinding, sessionID?: string) { + try { + Telemetry.track({ + type: "workspace_memory_contributed", + timestamp: Date.now(), + session_id: sessionID ?? Telemetry.getContext().sessionId, + workspace_id: binding.datamateId, + block_count: 1, + }) + } catch { + // telemetry never blocks a mirror + } +} +// altimate_change end + /** Cloud operations for one logical block, run one at a time. * * Both callers are fire-and-forget from the store's write and delete paths, so @@ -958,8 +993,11 @@ export async function flushPendingMirrors(timeoutMs = 30_000): Promise { /** Mirror one block. Safe to call unconditionally — returns immediately when * workspaces are disabled, the project is unbound, or the workspace has memory * disabled. */ -export async function mirrorBlock(block: MemoryBlock, directory?: string): Promise { +export async function mirrorBlock(block: MemoryBlock, directory?: string, sessionID?: string): Promise { if (!isEnabled()) return + // altimate_change — the writer's session: the one the memory tool passed, else the context's read + // before the queue (the context can be another session's by the time the mirror lands). + const writer = sessionID ?? Telemetry.getContext().sessionId // Queued BEFORE the binding lookup, not after. Both are async, so resolving // them first let two operations on one block reach `serialize` in the // opposite order to the writes that triggered them. @@ -984,7 +1022,7 @@ export async function mirrorBlock(block: MemoryBlock, directory?: string): Promi return } if (memory !== "enabled") return - await push(block, binding, undefined, directory) + await push(block, binding, undefined, directory, writer) }) return tracked(task) } @@ -992,11 +1030,7 @@ export async function mirrorBlock(block: MemoryBlock, directory?: string): Promi /** Archive a block's cloud record rather than deleting it, so the workspace * keeps the history. Only this client filters the marker — other readers do * not — so an archived record stays visible elsewhere. */ -export async function archiveBlock( - scope: "global" | "project", - blockId: string, - directory?: string, -): Promise { +export async function archiveBlock(scope: "global" | "project", blockId: string, directory?: string): Promise { if (!isEnabled()) return // Queued behind any in-flight mirror for the same block, so a delete cannot // run before the create it is meant to undo. The binding lookup happens @@ -1015,11 +1049,7 @@ export async function archiveBlock( ) } -async function archiveNow( - scope: "global" | "project", - blockId: string, - binding: CachedBinding, -): Promise { +async function archiveNow(scope: "global" | "project", blockId: string, binding: CachedBinding): Promise { const key = indexKey({ scope, blockId, @@ -1193,9 +1223,7 @@ async function pollMemoKey(binding: CachedBinding): Promise { * * An "unknown" is never memoized: the next tick should ask again rather than * inherit a network blip. */ -export async function memoryEnabledForPoller( - binding: CachedBinding, -): Promise<"enabled" | "disabled" | "unknown"> { +export async function memoryEnabledForPoller(binding: CachedBinding): Promise<"enabled" | "disabled" | "unknown"> { // The scoped memo is the poller's only cache. `memoryEnabledCache` is keyed by // bare workspace id — right for the write path, which is tenant-bound by its // credentials — but consulting it here reopened a 60s cross-tenant window the @@ -1204,8 +1232,7 @@ export async function memoryEnabledForPoller( // minutes per tenant is the price, and `memoryStatus` still warms both. const key = await pollMemoKey(binding) const memo = pollMemo.get(key) - if (memo && Date.now() - memo.at < (memo.status === "unknown" ? POLL_UNKNOWN_TTL_MS : POLL_TTL_MS)) - return memo.status + if (memo && Date.now() - memo.at < (memo.status === "unknown" ? POLL_UNKNOWN_TTL_MS : POLL_TTL_MS)) return memo.status // The in-flight ask is memoized too, not only the settled answer. Two // refreshes overlapping on a cold memo — a remount while a slow one is // still out — both saw it empty and both put a request on the wire. @@ -1214,7 +1241,7 @@ export async function memoryEnabledForPoller( const ask = (async () => { try { const status = await memoryStatus(binding, { fresh: true }) - const answer = status === "error" ? ("unknown" as const) : status + const answer = typeof status === "string" ? status : ("unknown" as const) pollMemo.set(key, { at: Date.now(), status: answer }) return answer } finally { @@ -1340,15 +1367,18 @@ export async function backfill( // ``gated`` says the sweep never ran, as opposed to running and storing // nothing. A caller recording "this binding is seeded" must be able to tell // those apart: memory being off is not a completed seed. - if (!isEnabled()) return { ok: 0, failed: 0, skipped: 0, declined: 0, deferred: 0, gated: true, gateReason: "local-off" } + if (!isEnabled()) + return { ok: 0, failed: 0, skipped: 0, declined: 0, deferred: 0, gated: true, gateReason: "local-off" } // The bind path passes the binding it just recorded; there is no ambient // instance to resolve one from on the `link` subcommand. const binding = explicitBinding ?? (await currentBinding()) if (!binding) return { ok: 0, failed: 0, skipped: blocks.length, declined: 0, deferred: 0, gated: true, gateReason: "unbound" } const status = await memoryStatus(binding) - if (status !== "enabled") - return { ok: 0, failed: 0, skipped: blocks.length, declined: 0, deferred: 0, gated: true, gateReason: status } + if (status !== "enabled") { + const gateReason = typeof status === "string" ? status : status.status + return { ok: 0, failed: 0, skipped: blocks.length, declined: 0, deferred: 0, gated: true, gateReason } + } const index = await readIndex() const { pending, skipped } = partitionPending(blocks, binding, index) @@ -1370,10 +1400,7 @@ export async function backfill( log.info("workspace memory backfill starting", { pending: pending.length, skipped }) const result = await runQueue( pending, - (item) => - serialize(item.block.scope, item.block.id, () => - push(item.block, item.binding, known, sweepDirectory), - ), + (item) => serialize(item.block.scope, item.block.id, () => push(item.block, item.binding, known, sweepDirectory)), BACKFILL_CONCURRENCY, ) // `skipped` combines blocks filtered before the queue (already synced, or @@ -1388,7 +1415,10 @@ export async function backfill( function originLabel(meta: Record): string | undefined { const remote = typeof meta.repo_remote === "string" ? meta.repo_remote : undefined if (remote) { - const trimmed = remote.replace(/[/]+$/, "").replace(/\.git$/, "").replace(/[/]+$/, "") + const trimmed = remote + .replace(/[/]+$/, "") + .replace(/\.git$/, "") + .replace(/[/]+$/, "") const last = trimmed.split(/[/:]/).pop() if (last) return last } @@ -1484,10 +1514,7 @@ export async function hydrate(sessionID: string): Promise { /** Wait for a session's in-flight hydration, capped. Resolves immediately once * the fetch has settled, so only the first injection of a session pays. */ -export async function whenHydrated( - sessionID: string, - timeoutMs: number = HYDRATION_WAIT_MS, -): Promise { +export async function whenHydrated(sessionID: string, timeoutMs: number = HYDRATION_WAIT_MS): Promise { const state = sessions.get(sessionID) const pending = state?.hydration if (!pending || !state) return @@ -1528,7 +1555,7 @@ type LoadOutcome = ( } | { status: "unlinked"; /** The server confirmed it, rather than a lookup failing. */ confirmed: boolean } | { status: "disabled" } - | { status: "error" } + | { status: "error"; /** What failed, for the sync record and its reason class. */ error: string } ) & { epoch?: string dir?: string | null @@ -1570,12 +1597,12 @@ async function loadWorkspaceMemory(directory?: string): Promise { stable = before === epoch } vouched = epoch - if (!stable) return { status: "error", epoch, dir, ...stamp } + if (!stable) return { status: "error", error: "the workspace link changed while loading", epoch, dir, ...stamp } if (!binding) return { status: "unlinked", confirmed, epoch, dir, ...stamp } const datamateId = binding.datamateId resolved = datamateId const enabled = await memoryStatus(binding) - if (enabled === "error") return { status: "error", epoch, dir, datamateId, ...stamp } + if (typeof enabled !== "string") return { status: "error", error: enabled.error, epoch, dir, datamateId, ...stamp } if (enabled === "disabled") return { status: "disabled", epoch, dir, datamateId, ...stamp } const ownProject = projectIdentityFor(binding) @@ -1630,7 +1657,7 @@ async function loadWorkspaceMemory(directory?: string): Promise { log.warn("workspace memory load failed", { err: String(err) }) // Stamped like any other outcome: without an epoch the session would reload (and make // the prompt wait) on every turn for as long as the service is down. - return { status: "error", epoch: vouched, dir, datamateId: resolved, ...stamp } + return { status: "error", error: failureText(err), epoch: vouched, dir, datamateId: resolved, ...stamp } } } @@ -1676,7 +1703,9 @@ function recordSyncState(outcome: LoadOutcome): void { if (retire) return SyncState.forget(dir, "memory", { isCurrent, before: outcome.observedAt }) if (outcome.datamateId === undefined) return const result = - outcome.status === "loaded" ? { items: outcome.items } : { error: "could not load workspace memory" } + outcome.status === "loaded" + ? { items: outcome.items } + : { error: `could not load workspace memory: ${"error" in outcome ? outcome.error : outcome.status}` } return SyncState.record(dir, outcome.datamateId, "memory", result, { isCurrent, scope: outcome.scope, diff --git a/packages/opencode/src/altimate/workspace/precedence.ts b/packages/opencode/src/altimate/workspace/precedence.ts index ff070b578e..bb3d9f1fa1 100644 --- a/packages/opencode/src/altimate/workspace/precedence.ts +++ b/packages/opencode/src/altimate/workspace/precedence.ts @@ -48,6 +48,7 @@ // that cache sooner, so it makes the next re-derivation see the change earlier — but the // per-turn re-derivation is the mechanism, not the notification. import { Config } from "@/config/config" +import { resetEngineTelemetryForTests, trackEngineError, trackPrecedence } from "./engine-telemetry" import { AppRuntime } from "@/effect/app-runtime" import { EventV2Bridge } from "@/event-v2-bridge" import { TuiEvent } from "@/server/tui-event" @@ -65,6 +66,8 @@ import { settledOutcome, type EntryLike, type Outcome, + linkedWorkspace, + linkedWorkspaceKey, } from "./engine-overlay" import { readLocalBindingScopedStrict, resolvePinnedBindingForRouting } from "./state" import { liveBridge } from "./engine-probes" @@ -532,10 +535,22 @@ export async function refresh( result = await derive(sessionID, tools) } catch (err) { log.warn("precedence could not be derived; running locally with a notice", { sessionID, err: String(err) }) + trackEngineError("precedence", String(err), sessionID) result = EMPTY("derive-failed") } if (ruleset) result.ruleset = ruleset remember(sessionID, result) + // telemetry: only when the snapshot (or the local connections it shadows) changed + trackPrecedence(sessionID, result, { + escapeHatch: escapeHatchOn(), + // The registry is read only when something can be shadowed: a disabled snapshot shadows nothing. + connections: result.enabled ? countShadowedConnections(result) : 0, + // Types the session can route through the engine, as the inventory line counts them. + types: result.enabled ? servedInventory(result).length : 0, + identity: linkedWorkspaceKey(), + // Named explicitly for disabled snapshots that carry no binding of their own. + workspaceId: Number(linkedWorkspace()?.id) || undefined, + }) // Mechanism 6 — say once, per session, what is now served where. Silence is the one // thing this design does not allow, but repeating it every turn would be noise. // A session that never had routing is told nothing — there is nothing to say. But a @@ -689,6 +704,7 @@ export function announcedSessionCount(): number { } export function resetForTests(): void { + resetEngineTelemetryForTests() bySession.clear() incarnations.clear() announced.clear() @@ -1141,7 +1157,7 @@ export function inventoryLine(precedence: Precedence): string { return `Workspace integrations — ${parts.join("; ")}. ${shadowedCount} local connection${shadowedCount === 1 ? "" : "s"} shadowed.` } -function countShadowedConnections(precedence: Precedence): number { +export function countShadowedConnections(precedence: Precedence): number { try { return Registry.list().warehouses.filter((w) => { const type = canonicalType(w.type) diff --git a/packages/opencode/src/altimate/workspace/skill-publish.ts b/packages/opencode/src/altimate/workspace/skill-publish.ts index 412f287f8c..098456f572 100644 --- a/packages/opencode/src/altimate/workspace/skill-publish.ts +++ b/packages/opencode/src/altimate/workspace/skill-publish.ts @@ -594,7 +594,8 @@ async function skillOwner(publicId: string): Promise { const detail = await altimateRequest<{ created_by?: unknown; skill?: { created_by?: unknown } }>( "GET", `/${encodeURIComponent(publicId)}`, - { base: SKILLS_BASE }, + // altimate_change - a 404 is the answer "the skill is gone", not a failed call + { base: SKILLS_BASE, expectedStatuses: [404] }, ) const raw = detail?.skill?.created_by ?? detail?.created_by return typeof raw === "number" ? raw : null diff --git a/packages/opencode/src/altimate/workspace/starter-telemetry.ts b/packages/opencode/src/altimate/workspace/starter-telemetry.ts new file mode 100644 index 0000000000..dd3ae3c9de --- /dev/null +++ b/packages/opencode/src/altimate/workspace/starter-telemetry.ts @@ -0,0 +1,53 @@ +// altimate_change - new file +// +// `starter_action_picked`: what the user chose from the workspace starter in the TUI. Kept apart from +// `starter.ts`, which the TUI imports for its type only: building a starter loads the skill module and +// the workspace service client, which the TUI's thread does not otherwise load. +import { Telemetry } from "@/altimate/telemetry" +import type { Starter } from "./starter" + +export type StarterAction = "prompt" | "open" | "done" + +export interface StarterPick { + action: StarterAction + /** Which suggested prompt, 0-based; only for `prompt`. */ + promptIndex?: number +} + +/** A starter dialog option's value (`prompt:`, `open`, `done`) as the action it picks; null for any + * other value. Pure. */ +export function starterPick(value: string): StarterPick | null { + if (value === "open" || value === "done") return { action: value } + const match = /^prompt:(\d+)$/.exec(value) + if (!match) return null + const index = Number(match[1]) + return Number.isSafeInteger(index) ? { action: "prompt", promptIndex: index } : null +} + +/** The row for a pick: the action, the prompt's index for a prompt, and the workspace. Never the prompt's text. */ +export function starterActionPickedEvent(workspaceId: number, pick: StarterPick, sessionID?: string): Telemetry.Event { + return { + type: "starter_action_picked", + timestamp: Date.now(), + session_id: sessionID ?? Telemetry.getContext().sessionId, + workspace_id: workspaceId, + action: pick.action, + ...(pick.action === "prompt" && pick.promptIndex !== undefined ? { prompt_index: pick.promptIndex } : {}), + } +} + +/** Track one selection from a starter dialog. `starter` is what the dialog shows; null while it has not + * arrived, when the dialog is the bare link confirmation and a selection is no starter outcome. The row names + * the starter's own workspace, the one its `starter_shown` row named. Nothing for a value that is not a + * starter action or a workspace id that is not a positive integer. Never throws. */ +export function trackStarterActionPicked(starter: Pick | null, value: string): boolean { + try { + const workspaceId = starter?.workspaceId + const pick = starterPick(value) + if (!pick || workspaceId === undefined || !Number.isInteger(workspaceId) || workspaceId <= 0) return false + Telemetry.track(starterActionPickedEvent(workspaceId, pick)) + return true + } catch { + return false + } +} diff --git a/packages/opencode/src/altimate/workspace/starter.ts b/packages/opencode/src/altimate/workspace/starter.ts index 7532073926..6d4db78e0e 100644 --- a/packages/opencode/src/altimate/workspace/starter.ts +++ b/packages/opencode/src/altimate/workspace/starter.ts @@ -5,6 +5,7 @@ // `GET /altimate/workspace/starter`. Fixed in format and built from the same data as the prompt's // "What this Altimate Workspace provides" section, so it costs no model call (none of the user's // tokens) and reads the same every time. Anything not known is left out, never shown as "none". +import { Telemetry } from "@/altimate/telemetry" import { displayWorkspaceName } from "./workspace-name" import { workspaceContents, type WorkspaceContents } from "./contents" @@ -18,6 +19,8 @@ export interface Starter { prompts: string[] /** The whole message as plain text, for a terminal or a chat bubble. */ text: string + /** The workspace the starter describes, so a client can report a pick against it. Set by `starterFor`. */ + workspaceId?: number } const MAX_PROMPTS = 3 @@ -33,10 +36,14 @@ function list(names: string[], max: number): string { } function skillsLine(skills: WorkspaceContents["skills"]): string | null { - if (skills === null) return "Skills: not synced to this project yet; run `altimate-code workspace refresh` to fetch them." + if (skills === null) + return "Skills: not synced to this project yet; run `altimate-code workspace refresh` to fetch them." if (skills === "unknown") return null if (skills.length === 0) return "Skills: none yet. Add them in the workspace and they sync here." - return `Skills (${skills.length}): ${list(skills.map((s) => s.name), MAX_LISTED_SKILLS)}` + return `Skills (${skills.length}): ${list( + skills.map((s) => s.name), + MAX_LISTED_SKILLS, + )}` } function knowledgeLine(k: WorkspaceContents["knowledge"]): string | null { @@ -86,7 +93,9 @@ function summaryOf(contents: WorkspaceContents): string { else if (k?.kind === "selected") parts.push(plural(k.names?.length ?? k.selected, "knowledge document")) else if (k?.kind === "off") parts.push("knowledge off") if (contents.integrations !== null) - parts.push(contents.integrations.length === 0 ? "no integrations" : `integrations: ${contents.integrations.join(", ")}`) + parts.push( + contents.integrations.length === 0 ? "no integrations" : `integrations: ${contents.integrations.join(", ")}`, + ) if (contents.memoryEnabled !== null) parts.push(contents.memoryEnabled ? "memory on" : "memory off") return parts.join(" · ") } @@ -138,10 +147,34 @@ export function wrapStarter(text: string, width: number): string { } /** The starter for a linked project. `directory` is the project key the link was recorded under. */ +// altimate_change start — the starter is the first thing a user sees of the workspace in a tool +/** `starter_shown`, with what the workspace provides as counts. Never throws. */ +export function trackStarterShown(workspaceId: number, contents: WorkspaceContents): void { + try { + Telemetry.track({ + type: "starter_shown", + timestamp: Date.now(), + session_id: Telemetry.getContext().sessionId, + workspace_id: workspaceId, + // Unknown is left out, not reported as none: the starter's own rule. + ...(Array.isArray(contents.skills) ? { skills: contents.skills.length } : {}), + ...(Array.isArray(contents.integrations) ? { integrations: contents.integrations.length } : {}), + knowledge: contents.knowledge?.kind ?? "unknown", + ...(typeof contents.memoryEnabled === "boolean" ? { memory_enabled: contents.memoryEnabled } : {}), + }) + } catch { + // telemetry never blocks the starter + } +} +// altimate_change end + export async function starterFor( directory: string, binding: { datamateId: number; datamateName: string }, waitMs = STARTER_WAIT_MS, ): Promise { - return renderStarter(binding.datamateName, await workspaceContents(directory, binding.datamateId, waitMs)) + const contents = await workspaceContents(directory, binding.datamateId, waitMs) + const starter = renderStarter(binding.datamateName, contents) + trackStarterShown(binding.datamateId, contents) + return { ...starter, workspaceId: binding.datamateId } } diff --git a/packages/opencode/src/altimate/workspace/sync-state.ts b/packages/opencode/src/altimate/workspace/sync-state.ts index 9e72ebc77b..5af5b9d17e 100644 --- a/packages/opencode/src/altimate/workspace/sync-state.ts +++ b/packages/opencode/src/altimate/workspace/sync-state.ts @@ -25,6 +25,7 @@ // run in its server worker. A refresh run on the TUI main thread, or by a separate CLI // process, publishes to no `/event` subscriber — the file is still written, and the TUI // sidebar and the status route read it from there. +import { Telemetry } from "@/altimate/telemetry" import { createHash, randomBytes } from "node:crypto" import { chmodSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs" import path from "node:path" @@ -275,6 +276,15 @@ function chain(file: string, work: () => Promise, kind: SyncKind): Promise return settled } +/** The reason class of a sync error message, for workspace_sync_failed. */ +export function syncFailureReason(error: string | undefined): "auth" | "not_found" | "network" | "other" { + const e = (error ?? "").toLowerCase() + if (/\b(401|403)\b|unauthori[sz]ed|forbidden|\bsign(ed)? in\b|credential/.test(e)) return "auth" + if (/\b404\b|not found|no longer exists|unknown workspace/.test(e)) return "not_found" + if (/timed? ?out|cannot reach|unreachable|econn|enotfound|network|socket|fetch failed/.test(e)) return "network" + return "other" +} + async function apply( directory: string, datamateId: number, @@ -330,6 +340,23 @@ async function apply( } const file: StoredFile = { version: 2, directory: canon, scope, datamateId, entity: next } writePrivate(filePath(directory, kind), file) + // altimate_change start — a kind flipping to error is a blocker the dashboard counts; a repeat + // while already in error is not. + if (next.status === "error" && previous?.status !== "error") { + try { + Telemetry.track({ + type: "workspace_sync_failed", + timestamp: Date.now(), + session_id: Telemetry.getContext().sessionId, + workspace_id: datamateId, + kind, + reason: syncFailureReason(outcome.error), + }) + } catch { + // telemetry never blocks the record + } + } + // altimate_change end const errorChanged = (previous?.status ?? "ok") !== next.status || previous?.error !== next.error if (!changes && !errorChanged) return diff --git a/packages/opencode/src/altimate/workspace/tool-call-telemetry.ts b/packages/opencode/src/altimate/workspace/tool-call-telemetry.ts new file mode 100644 index 0000000000..d2657c8abf --- /dev/null +++ b/packages/opencode/src/altimate/workspace/tool-call-telemetry.ts @@ -0,0 +1,85 @@ +// altimate_change — tool_call rows for the bound workspace's integration calls. +import { Telemetry } from "@/altimate/telemetry" +import { DATAMATE_KEY } from "@/altimate/datamate-transport" +import os from "os" +import { redact, type RedactContext } from "@/altimate/debug/report" + +let redactCtx: RedactContext | undefined +/** The home folder, user name and host name to strip from error text; gathered once, never throws. */ +function redactContext(): RedactContext { + if (redactCtx) return redactCtx + const tryGet = (f: () => string) => { + try { + return f() + } catch { + return undefined + } + } + redactCtx = { + home: tryGet(() => os.homedir()), + username: tryGet(() => os.userInfo().username) ?? process.env["USER"] ?? process.env["USERNAME"], + hostname: tryGet(() => os.hostname()), + } + return redactCtx +} + +/** True when the MCP client serving a tool is the engine the workspace overlay manages. */ +export function isWorkspaceClient(clientName: string): boolean { + return clientName === DATAMATE_KEY +} + +/** + * Track one MCP call that went through the bound workspace's engine. MCP tools never pass through + * the `Tool.define` wrapper that tracks registry tools, so this is where a workspace integration + * call becomes a `tool_call` row; a successful one also counts as an integration use for the + * session. Only the managed engine client (`datamate`) in a bound turn counts: a standalone + * `datamate-` server, or the key in an unbound turn, is not the workspace's. The turn's own + * workspace is passed in and named on the row, so a call that finishes after the turn was + * cancelled and replaced keeps the workspace it ran for. Returns whether the call was tracked. + */ +export function trackWorkspaceToolCall(input: { + key: string + clientName: string + sessionID: string + messageID: string + /** The workspace the calling turn resolved; undefined for an unbound turn. */ + workspaceId: number | undefined + /** The calling turn (from `setContext`); a success landing after it was replaced is not counted. */ + turn?: number + startedAt: number + outcome: "success" | "error" + error?: unknown +}): boolean { + if (!isWorkspaceClient(input.clientName)) return false + if (input.workspaceId === undefined) return false + try { + if (input.outcome === "success") Telemetry.noteWorkspaceUse(input.sessionID, "integrations", 1, "add", input.turn) + const error = + input.error === undefined + ? undefined + : redact( + Telemetry.maskString(input.error instanceof Error ? input.error.message : String(input.error)), + redactContext(), + ).slice(0, 500) + Telemetry.track({ + type: "tool_call", + timestamp: Date.now(), + session_id: input.sessionID, + message_id: input.messageID, + tool_name: input.key, + workspace_id: input.workspaceId, + tool_type: "mcp", + tool_category: Telemetry.categorizeToolName(input.key, "mcp"), + status: input.outcome, + via_workspace: true, + duration_ms: Date.now() - input.startedAt, + sequence_index: 0, + previous_tool: null, + ...(error !== undefined && { error }), + }) + } catch { + // telemetry never surfaces into the tool call + return false + } + return true +} diff --git a/packages/opencode/src/cli/cmd/run.ts b/packages/opencode/src/cli/cmd/run.ts index 3e3afe3a15..f6208dbbb5 100644 --- a/packages/opencode/src/cli/cmd/run.ts +++ b/packages/opencode/src/cli/cmd/run.ts @@ -71,7 +71,7 @@ import { SessionTermination } from "../../session/termination" // altimate_change start — upstream_fix: type-only import for the tracing-config cast (see tracer setup below) import type { ConfigV1 } from "@opencode-ai/core/v1/config/config" // altimate_change - render the workspace engine offer in an attached run -import { OFFER_COMMAND, installCommand } from "@/altimate/workspace/engine-offer" +import { OFFER_COMMAND, installCommand, trackPrintedOffer } from "@/altimate/workspace/engine-offer" import { OutputCollector } from "./run-output" // altimate_change end @@ -1345,6 +1345,9 @@ You are speaking to a non-technical business executive. Follow these rules stric `This workspace's integration tools need the local engine on the server. Install it there with: ${installCommand()}` + EOL, ) + // This process is the surface that showed the offer, so it reports `shown` (surface + // `bus`) from the fields the event carried; the server's publish is not a display. + if (sessionID) trackPrintedOffer((event.properties as { arguments?: unknown }).arguments, sessionID) continue } // altimate_change end diff --git a/packages/opencode/src/cli/cmd/skill.ts b/packages/opencode/src/cli/cmd/skill.ts index 53d1d86211..4b8d4ce86e 100644 --- a/packages/opencode/src/cli/cmd/skill.ts +++ b/packages/opencode/src/cli/cmd/skill.ts @@ -533,6 +533,7 @@ const SkillPublishCommand = cmd({ action: report.action, file_count: report.files, source: "cli", + workspace_id: report.datamateId, }) } catch {} } catch (err) { diff --git a/packages/opencode/src/memory/prompt.ts b/packages/opencode/src/memory/prompt.ts index 89c4cb2208..7cc30c852b 100644 --- a/packages/opencode/src/memory/prompt.ts +++ b/packages/opencode/src/memory/prompt.ts @@ -206,6 +206,8 @@ export namespace MemoryPrompt { let injectedCount = 0 const injectedTraining: MemoryBlock[] = [] const scopesSeen = new Set() + // altimate_change — blocks injected from the bound workspace's cloud memory + let workspaceBlocks = 0 // Group training blocks by kind for structured display const byKind = new Map() @@ -244,6 +246,13 @@ export namespace MemoryPrompt { injectedCount++ injectedTraining.push(item.block) scopesSeen.add(item.block.scope) + // altimate_change start — a project-scope block from the bound workspace's cloud memory counts as + // workspace scope; a global cloud block is the account's and applies everywhere + if (isRemote(item.block) && item.block.scope === "project") { + scopesSeen.add("workspace") + workspaceBlocks++ + } + // altimate_change end } } @@ -263,6 +272,12 @@ export namespace MemoryPrompt { used += needed injectedCount++ scopesSeen.add(item.block.scope) + // altimate_change start — as above, for memory blocks + if (isRemote(item.block) && item.block.scope === "project") { + scopesSeen.add("workspace") + workspaceBlocks++ + } + // altimate_change end } } } @@ -288,12 +303,21 @@ export namespace MemoryPrompt { Telemetry.track({ type: "memory_injection", timestamp: Date.now(), - session_id: Telemetry.getContext().sessionId, + // altimate_change — the injecting session, not the process context (another turn's in serve) + session_id: ctx?.sessionID ?? Telemetry.getContext().sessionId, + workspace_id: ctx?.workspaceId, // altimate_change — the turn's workspace, named explicitly block_count: injectedCount, total_chars: used, budget, scopes_used: [...scopesSeen], + // altimate_change start — workspace memory use, for memory_injection and the session's summary rows + ...(workspaceBlocks > 0 && { workspace_blocks: workspaceBlocks }), }) + // The turn injects on every step: the count is the most blocks any step injected, not a sum. + if (workspaceBlocks > 0 && ctx?.sessionID) { + Telemetry.noteWorkspaceUse(ctx.sessionID, "memory", workspaceBlocks, "max", ctx.turn) + } + // altimate_change end } return injectedCount > 0 ? result : "" diff --git a/packages/opencode/src/memory/store.ts b/packages/opencode/src/memory/store.ts index d6b2b7f46a..2665e033be 100644 --- a/packages/opencode/src/memory/store.ts +++ b/packages/opencode/src/memory/store.ts @@ -265,7 +265,11 @@ export namespace MemoryStore { }) } - export async function write(block: MemoryBlock): Promise<{ duplicates: MemoryBlock[] }> { + export async function write( + block: MemoryBlock, + // altimate_change — the writing session, for the workspace write-back's row + opts: { sessionID?: string } = {}, + ): Promise<{ duplicates: MemoryBlock[] }> { if (block.content.length > MEMORY_MAX_BLOCK_SIZE) { throw new Error( `Memory block "${block.id}" content exceeds maximum size of ${MEMORY_MAX_BLOCK_SIZE} characters (got ${block.content.length})`, @@ -308,7 +312,8 @@ export namespace MemoryStore { Telemetry.track({ type: "memory_operation", timestamp: Date.now(), - session_id: Telemetry.getContext().sessionId, + // altimate_change — the writing session when the caller passed it, else the context's + session_id: opts.sessionID ?? Telemetry.getContext().sessionId, operation: "write", scope: block.scope, block_id: block.id, @@ -327,7 +332,7 @@ export namespace MemoryStore { // local-existence check would then look for this block in the wrong tree // and skip it as deleted. const owningDirectory = safeDirectory() - void mirrorBlock(block, owningDirectory).catch((e) => { + void mirrorBlock(block, owningDirectory, opts.sessionID).catch((e) => { mirrorLog.warn("failed to mirror memory block to workspace", { id: block.id, scope: block.scope, diff --git a/packages/opencode/src/memory/tools/memory-extract.ts b/packages/opencode/src/memory/tools/memory-extract.ts index 1a316117bb..46fdb9b34b 100644 --- a/packages/opencode/src/memory/tools/memory-extract.ts +++ b/packages/opencode/src/memory/tools/memory-extract.ts @@ -50,7 +50,7 @@ export const MemoryExtractTool = Tool.define("altimate_memory_extract", { updated: now, citations: fact.citations, content: fact.content, - }) + }, { sessionID: ctx.sessionID }) // altimate_change — the writer's session, for the workspace row const action = existing ? "Updated" : "Created" results.push(` ✓ ${action} "${fact.id}" (${fact.scope})`) diff --git a/packages/opencode/src/memory/tools/memory-write.ts b/packages/opencode/src/memory/tools/memory-write.ts index 4613d9756d..439bc9ec0b 100644 --- a/packages/opencode/src/memory/tools/memory-write.ts +++ b/packages/opencode/src/memory/tools/memory-write.ts @@ -51,7 +51,7 @@ export const MemoryWriteTool = Tool.define("altimate_memory_write", { expires: args.expires, citations: args.citations, content: args.content, - }) + }, { sessionID: ctx.sessionID }) // altimate_change — the writer's session, for the workspace row const action = existing ? "Updated" : "Created" let output = `${action} memory block "${args.id}" in ${args.scope} scope.` diff --git a/packages/opencode/src/memory/types.ts b/packages/opencode/src/memory/types.ts index 8f3e34880c..61db9077ea 100644 --- a/packages/opencode/src/memory/types.ts +++ b/packages/opencode/src/memory/types.ts @@ -46,6 +46,10 @@ export const AGENT_TRAINING_RELEVANCE: Record { - // altimate_change start - put the chosen prompt in the input without sending it + // altimate_change start - the pick, once the starter is on screen (before, this is the bare confirmation) + trackStarterActionPicked(props.starter(), option.value) + // put the chosen prompt in the input without sending it if (option.value.startsWith("prompt:")) { fillPrompt(props.api, option.title) return @@ -523,6 +527,7 @@ function WorkspaceStarterDialog(props: { api: TuiPluginApi; starter: Starter }) ]} current={props.starter.prompts.length > 0 ? "prompt:0" : "done"} onSelect={(option) => { + trackStarterActionPicked(props.starter, option.value) if (option.value.startsWith("prompt:")) fillPrompt(props.api, option.title) else props.api.ui.dialog.clear() }} @@ -1784,6 +1789,8 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { // opened in our place. So completion reports through a toast when we are // gone, and only touches the dialog while we still own it. let mounted = true + // The offer was shown: reported from the surface that rendered it, not from the dispatch. + onMount(() => trackInstallPrompt("shown", props.offer, { surface: "dialog", nodeMajor: props.nodeMajor })) onCleanup(() => { mounted = false // Release the single-offer latch however this dialog goes away — chosen, @@ -1870,6 +1877,8 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { const performInstall = async () => { const result = await installEngine() if (!result.ok) { + trackInstallPrompt("install_failed", props.offer) + trackEngineError("install", result.error) installing = false if (!mounted) { // Dismissed mid-install: the failed-phase rows have nowhere to render, @@ -1885,6 +1894,7 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { setPhase("failed") return } + trackInstallPrompt("installed", props.offer) // Only clear a dialog we still own — by now the user may have opened // another, and clearing the stack would take theirs down instead. if (mounted) props.api.ui.dialog.clear() @@ -1905,21 +1915,12 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { const copyCommand = () => { const cmd = command() void (async () => { - try { - await clipboard.write?.(cmd) - // A resolved write is NOT proof of a copy. The host's writer picks - // xclip/xsel on Linux and otherwise falls back to clipboardy, and it - // swallows backend failures (`.catch(() => undefined)`), so on the many - // Linux/WSL boxes with neither tool installed the write silently does - // nothing. Read back and compare before claiming success. (Caught by - // E2E: the toast said "Copied:" while the clipboard was untouched.) - const back = await clipboard.read?.() - if (back?.data.trim() === cmd) { - props.api.ui.toast({ variant: "info", message: `Copied: ${cmd}` }) - return - } - } catch { - // Unreadable or unwritable clipboard — fall through and show it. + if (await confirmedCopy(clipboard, cmd)) { + // `copied` only once the read-back confirms the clipboard holds the command: the + // selection itself proves nothing (an unconfirmed copy tracks no row). + trackInstallPrompt("copied", props.offer) + props.api.ui.toast({ variant: "info", message: `Copied: ${cmd}` }) + return } props.api.ui.toast({ variant: "warning", @@ -1947,8 +1948,13 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { if (option.value === "install") { if (installing) return installing = true + trackInstallPrompt("accepted", props.offer) void runInstall().catch((err) => { const message = err instanceof Error ? err.message : String(err) + // A rejection (a throw past installEngine's own result) is the same failure to the + // funnel as a reported one. + trackInstallPrompt("install_failed", props.offer) + trackEngineError("install", message) installing = false if (!mounted) { // Same as a reported failure after dismissal: the rows are gone, @@ -1971,6 +1977,7 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { return } if (option.value === "skip") { + trackInstallPrompt("declined", props.offer) recordEngineSkip(props.api, props.offer.workspaceId, props.latchScope, Date.now()) } props.api.ui.dialog.clear() @@ -1979,6 +1986,25 @@ function EngineInstallOfferDialog(props: EngineOfferProps) { ) } +/** Write `text` to the clipboard and read it back: true only when the clipboard then holds it. + * + * A resolved write is NOT proof of a copy. The host's writer picks xclip/xsel on Linux and + * otherwise falls back to clipboardy, and it swallows backend failures (`.catch(() => undefined)`), + * so on the many Linux/WSL boxes with neither tool installed the write silently does nothing. + * (Caught by E2E: the toast said "Copied:" while the clipboard was untouched.) An unreadable or + * unwritable clipboard is the same unconfirmed outcome. */ +export async function confirmedCopy(clipboard: Pick, text: string) { + // No writer means nothing was copied, whatever the clipboard happens to hold. + if (!clipboard.write) return false + try { + await clipboard.write(text) + const back = await clipboard.read?.() + return back?.data.trim() === text + } catch { + return false + } +} + /** Show the offer unless the 7-day latch suppresses it. * * The overlay raises this as a bare command over the event bus — it cannot hand diff --git a/packages/opencode/src/server/server.ts b/packages/opencode/src/server/server.ts index a917f93d6a..dddd3299ac 100644 --- a/packages/opencode/src/server/server.ts +++ b/packages/opencode/src/server/server.ts @@ -1348,6 +1348,7 @@ export namespace Server { action: report.action, file_count: report.files, source: "serve", + workspace_id: report.datamateId, }) } catch {} return c.json({ ok: true as const, report, message: describePublish(report) }) diff --git a/packages/opencode/src/server/tui-event.ts b/packages/opencode/src/server/tui-event.ts index 93d3f7e6df..c7e62862d4 100644 --- a/packages/opencode/src/server/tui-event.ts +++ b/packages/opencode/src/server/tui-event.ts @@ -36,6 +36,8 @@ export const TuiEvent = { // same stream: the session it was raised for lets that run print only its // own offer, not another session's in the same directory. sessionID: Schema.optional(Schema.String), + // The offer's telemetry fields (JSON), for the surface that prints it to report `shown` with. + arguments: Schema.optional(Schema.String), // altimate_change end }, }), diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index f6e2c0c0f8..ecedc18104 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -114,10 +114,26 @@ import { Config } from "../config/config" import { Tracer } from "../altimate/observability/tracing" // altimate_change start — stamp an authoritative tool source + humanized MCP title import { stampRegistryToolSource, describeMcpTool } from "../altimate/tool-source" +import { trackWorkspaceToolCall } from "../altimate/workspace/tool-call-telemetry" // altimate_change — workspace integration rows // altimate_change end // altimate_change end import { Telemetry } from "@/telemetry" // altimate_change — session telemetry import { ErrorDetail } from "@/altimate/telemetry/error-detail" // altimate_change — error_fingerprint payload +// altimate_change start — the session's bound workspace rides on every telemetry row +import { readLocalBinding } from "@/altimate/workspace/state" + +/** The session's workspace use, as session_end fields. Read only: the row is tracked first, then the + * turn ends its telemetry entries (`Telemetry.endSession`), so the row itself still carries the + * workspace. */ +function workspaceUseFields(sessionID: string, turn: number) { + const use = Telemetry.workspaceUseOf(sessionID, turn) + return { + workspace_skills_used: use.skills, + workspace_memory_blocks: use.memory, + workspace_integration_calls: use.integrations, + } +} +// altimate_change end import { TraceContext } from "@/altimate/observability/trace-context" // altimate_change — client trace per turn import * as OnboardingTelemetry from "@/altimate/telemetry/onboarding" // altimate_change — onboarding funnel // altimate_change start — keyless public Zen predicate (flat module) @@ -763,7 +779,19 @@ export namespace SessionPrompt { throw e } // altimate_change end - Telemetry.setContext({ sessionId: sessionID, projectId: Instance.project?.id ?? "" }) + // altimate_change start — the workspace this project is linked to (local binding cache, no + // network), named by the rows of this turn that carry workspace_id. + const boundWorkspace = await readLocalBinding(Instance.directory).catch(() => null) + const telemetryTurn = Telemetry.setContext({ + sessionId: sessionID, + projectId: Instance.project?.id ?? "", + }) + // The turn's own workspace, named on the rows this loop emits at its end: a cancelled turn + // unwinding after its replacement started must not take the replacement's entry. + // Always keyed, undefined for an unbound turn: a row that carries the key is never stamped from + // the session's entry, and the envelope drops an undefined value. + const turnWorkspace = { workspace_id: boundWorkspace?.datamateId } + // altimate_change end // altimate_change start — the learn kill switch disables new delivery, while retaining trusted note cleanup. // An unused project pays only an existence check; do not load stores, migrate, // create state, or even schedule async learning work on that default-off path. @@ -827,13 +855,18 @@ export namespace SessionPrompt { emergencySessionEndFired = true Telemetry.track({ type: "session_end", + ...turnWorkspace, // altimate_change — this turn's workspace, not the session's current entry timestamp: Date.now(), session_id: sessionID, total_cost: sessionTotalCost, total_tokens: sessionTotalTokens, tool_call_count: toolCallCount, duration_ms: Date.now() - sessionStartTime, + // altimate_change start — what of the bound workspace this session used + ...workspaceUseFields(sessionID, telemetryTurn), + // altimate_change end }) + Telemetry.endSession(sessionID, telemetryTurn) // altimate_change — after the row, this turn only } process.once("beforeExit", emergencySessionEnd) process.once("exit", emergencySessionEnd) @@ -1271,6 +1304,10 @@ export namespace SessionPrompt { sessionID: sessionID, abort, callID: part.callID, + // altimate_change start — the turn's workspace and token, for the task's own rows + workspaceId: boundWorkspace?.datamateId, + turn: telemetryTurn, + // altimate_change end extra: { bypassAgentCheck: true }, // altimate_change start — pass effective permission rules to search result filtering permissions: PermissionNext.merge(taskAgent.permission, session.permission ?? []), @@ -1320,6 +1357,7 @@ export namespace SessionPrompt { sessionID, callID: part.id, args: taskArgs, + workspaceId: boundWorkspace?.datamateId, // altimate_change — the turn's workspace, for plugin rows }, result, ) @@ -1559,6 +1597,7 @@ export namespace SessionPrompt { // Cap reached — track and inject a synthetic hint so the LLM informs the user Telemetry.track({ type: "plan_revision", + ...turnWorkspace, // altimate_change — this turn's workspace, named explicitly timestamp: Date.now(), session_id: sessionID, revision_number: planRevisionCount, @@ -1643,6 +1682,7 @@ export namespace SessionPrompt { const action = isRejection ? "reject" : isApproval ? "approve" : "refine" Telemetry.track({ type: "plan_revision", + ...turnWorkspace, // altimate_change — this turn's workspace, named explicitly timestamp: Date.now(), session_id: sessionID, revision_number: planRevisionCount, @@ -1723,6 +1763,8 @@ export namespace SessionPrompt { messages: msgs, // altimate_change start — file rules append only to this tool result lessons, + workspaceId: boundWorkspace?.datamateId, + turn: telemetryTurn, // altimate_change end }), { step, agent: agent.name }, @@ -1785,6 +1827,7 @@ export namespace SessionPrompt { // "build" sessions). See the helper at the top of this namespace. Telemetry.track({ type: "session_start", + ...turnWorkspace, // altimate_change — this turn's workspace, named explicitly timestamp: Date.now(), session_id: sessionID, model_id: model.id, @@ -1836,6 +1879,7 @@ export namespace SessionPrompt { ) ?? "unknown" Telemetry.track({ type: "task_classified", + ...turnWorkspace, // altimate_change — this turn's workspace, named explicitly timestamp: Date.now(), session_id: sessionID, intent: intent as any, @@ -1869,7 +1913,7 @@ export namespace SessionPrompt { OnboardingTelemetry.isOnboardingSession(sessionID) && OnboardingTelemetry.claimFirstPrompt(sessionID) ) { - void OnboardingTelemetry.emit({ type: "first_prompt_sent" }, sessionID) + void OnboardingTelemetry.emit({ type: "first_prompt_sent", ...turnWorkspace }, sessionID) } // altimate_change end // altimate_change end — session start telemetry @@ -1911,6 +1955,9 @@ export namespace SessionPrompt { agent: agent.name, disableTraining: Flag.ALTIMATE_DISABLE_TRAINING, sessionID, + // altimate_change — the turn's workspace and token, for the injection row and its count + turn: telemetryTurn, + workspaceId: boundWorkspace?.datamateId, }) // altimate_change end // altimate_change start — workspace identity. @@ -2366,6 +2413,7 @@ export namespace SessionPrompt { // (verdict enums, counters, model names) passes through. Telemetry.track({ type: "validator_check", + ...turnWorkspace, // altimate_change — this turn's workspace, named explicitly timestamp: Date.now(), session_id: sessionID, validator_name: validator.name, @@ -2432,6 +2480,7 @@ export namespace SessionPrompt { }) Telemetry.track({ type: "validator_retries_exhausted", + ...turnWorkspace, // altimate_change — this turn's workspace, named explicitly timestamp: Date.now(), session_id: sessionID, step, @@ -2531,6 +2580,7 @@ export namespace SessionPrompt { // altimate_change start — emit quality signal, tool chain, and error fingerprint events Telemetry.track({ type: "task_outcome_signal", + ...turnWorkspace, // altimate_change — this turn's workspace, not the session's current entry timestamp: Date.now(), session_id: sessionID, signal: Telemetry.deriveQualitySignal(outcome), @@ -2538,11 +2588,14 @@ export namespace SessionPrompt { step_count: step, duration_ms: Date.now() - sessionStartTime, last_tool_category: lastToolCategory || "none", + // altimate_change — whether workspace skills, memory or integrations were in play + workspace_context_used: Object.values(Telemetry.workspaceUseOf(sessionID, telemetryTurn)).some((n) => n > 0), }) // Tool chain effectiveness — aggregated tool sequence + outcome if (toolChain.length > 0) { Telemetry.track({ type: "tool_chain_outcome", + ...turnWorkspace, // altimate_change — this turn's workspace, not the session's current entry timestamp: Date.now(), session_id: sessionID, chain: JSON.stringify(toolChain), @@ -2560,7 +2613,7 @@ export namespace SessionPrompt { } // Error fingerprints — one event per unique error (capped at 20) for (const err of errorRecords.slice(0, 20)) { - Telemetry.track(ErrorDetail.buildErrorFingerprintEvent(sessionID, err)) + Telemetry.track({ ...ErrorDetail.buildErrorFingerprintEvent(sessionID, err), ...turnWorkspace }) // altimate_change — this turn's workspace, not the session's current entry } // altimate_change end — emit quality signal, tool chain, and error fingerprint events // altimate_change start — populate agent_outcome diagnostic fields @@ -2584,6 +2637,7 @@ export namespace SessionPrompt { // altimate_change end Telemetry.track({ type: "agent_outcome", + ...turnWorkspace, // altimate_change — this turn's workspace, not the session's current entry timestamp: Date.now(), session_id: sessionID, // altimate_change start — route through normalizeAgentName (shared with @@ -2609,13 +2663,18 @@ export namespace SessionPrompt { process.off("exit", emergencySessionEnd) Telemetry.track({ type: "session_end", + ...turnWorkspace, // altimate_change — this turn's workspace, not the session's current entry timestamp: Date.now(), session_id: sessionID, total_cost: sessionTotalCost, total_tokens: sessionTotalTokens, tool_call_count: toolCallCount, duration_ms: Date.now() - sessionStartTime, + // altimate_change start — what of the bound workspace this session used + ...workspaceUseFields(sessionID, telemetryTurn), + // altimate_change end }) + Telemetry.endSession(sessionID, telemetryTurn) // altimate_change — after the row, this turn only } await Telemetry.shutdown() // altimate_change end @@ -2717,6 +2776,10 @@ export namespace SessionPrompt { messages: MessageV2.WithParts[] // altimate_change start — optional for callers outside the main prompt loop lessons?: LessonDelivery + /** The workspace the calling turn resolved, for the rows its MCP calls emit, and the turn itself, + * so a call landing after the turn was replaced does not count toward the replacement. */ + workspaceId?: number + turn?: number // altimate_change end }) { using _ = log.time("resolveTools") @@ -2730,6 +2793,10 @@ export namespace SessionPrompt { abort: options.abortSignal!, messageID: input.processor.message.id, callID: options.toolCallId, + // altimate_change start — the turn's workspace and token, for the rows the tool emits + workspaceId: input.workspaceId, + turn: input.turn, + // altimate_change end // altimate_change start — batch inner calls reuse the prompt's lesson delivery state extra: { model: input.model, bypassAgentCheck: input.bypassAgentCheck, lessons: input.lessons }, // altimate_change end @@ -2845,6 +2912,7 @@ export namespace SessionPrompt { sessionID: ctx.sessionID, callID: ctx.callID, args, + workspaceId: input.workspaceId, // altimate_change — the turn's workspace, for plugin rows }, // altimate_change start — plugins observe the source-stamped output stamped, @@ -2911,7 +2979,36 @@ export namespace SessionPrompt { ) // altimate_change end - const result = await execute(args, opts) + // altimate_change start — a workspace integration call is tracked here, where MCP tools run + const startedAt = Date.now() + let result + try { + result = await execute(args, opts) + } catch (err) { + trackWorkspaceToolCall({ + key, + clientName, + sessionID: ctx.sessionID, + messageID: ctx.messageID, + workspaceId: input.workspaceId, + turn: input.turn, + startedAt, + outcome: "error", + error: err, + }) + throw err + } + trackWorkspaceToolCall({ + key, + clientName, + sessionID: ctx.sessionID, + messageID: ctx.messageID, + workspaceId: input.workspaceId, + turn: input.turn, + startedAt, + outcome: (result as { isError?: boolean }).isError === true ? "error" : "success", + }) + // altimate_change end await Plugin.trigger( "tool.execute.after", @@ -2920,6 +3017,7 @@ export namespace SessionPrompt { sessionID: ctx.sessionID, callID: opts.toolCallId, args, + workspaceId: input.workspaceId, // altimate_change — the turn's workspace, for plugin rows }, result, ) @@ -3237,6 +3335,8 @@ export namespace SessionPrompt { const model = await Provider.getModel(info.model.providerID, info.model.modelID) const readCtx: Tool.Context = { sessionID: input.sessionID, + // altimate_change — the project's binding, so the attachment read's tool rows name the workspace + workspaceId: (await readLocalBinding(Instance.directory).catch(() => null))?.datamateId, abort: new AbortController().signal, agent: input.agent!, messageID: info.id, diff --git a/packages/opencode/src/tool/batch.ts b/packages/opencode/src/tool/batch.ts index 9dc732347a..7f6122b2c9 100644 --- a/packages/opencode/src/tool/batch.ts +++ b/packages/opencode/src/tool/batch.ts @@ -23,6 +23,9 @@ function toEffectContext(ctx: LegacyContext, callID: string): Tool.Context { agent: ctx.agent, abort: ctx.abort, callID, + // the turn's workspace and token, so the inner tools' rows name it + workspaceId: ctx.workspaceId, + turn: ctx.turn, extra: ctx.extra, messages: ctx.messages, // altimate_change start — preserve search result permissions in nested batch tools diff --git a/packages/opencode/src/tool/skill.ts b/packages/opencode/src/tool/skill.ts index a33dd49a34..26271bb804 100644 --- a/packages/opencode/src/tool/skill.ts +++ b/packages/opencode/src/tool/skill.ts @@ -19,11 +19,16 @@ import { selectSkillsWithLLM } from "../altimate/skill-selector" import { Telemetry } from "../altimate/telemetry" import os from "os" import { isBuiltinSkillLocation } from "../altimate/skill-source" +import { Instance } from "../project/instance" const MAX_DISPLAY_SKILLS = 50 // altimate_change start — classifySkillSource helper for skill telemetry + source badge -export function classifySkillSource(location: string): "builtin" | "global" | "project" { +export function classifySkillSource( + location: string, + /** This project's root; with it, only its own managed folder is the workspace's. */ + projectRoot?: string, +): "builtin" | "global" | "project" | "workspace" { // Normalize separators so `.altimate/builtin` / homedir prefix match on Windows too. const normalized = location.replace(/\\/g, "/") // Embedded skills load with a `builtin:/SKILL.md` location and Altimate's @@ -33,6 +38,14 @@ export function classifySkillSource(location: string): "builtin" | "global" | "p // `node_modules/` isn't tagged as Altimate. // altimate_change - the same predicate the skill listing labels with, so the two cannot drift apart if (isBuiltinSkillLocation(location)) return "builtin" + // altimate_change start — skills synced from the bound workspace live under the managed + // `_workspace` folder of this project; they are the workspace's, not the project's. Another + // project's managed folder is not this session's workspace. + if (projectRoot !== undefined) { + const root = projectRoot.replace(/\\/g, "/").replace(/\/+$/, "") + if (normalized.startsWith(`${root}/.altimate-code/skill/_workspace/`)) return "workspace" + } else if (/\/\.altimate-code\/skill\/_workspace\//.test(normalized)) return "workspace" + // altimate_change end if (normalized.startsWith(os.homedir().replace(/\\/g, "/"))) return "global" return "project" } @@ -256,7 +269,13 @@ export const SkillTool = Tool.define("skill", async (ctx) => { // altimate_change end // altimate_change start — classify origin once, reused for telemetry and the source badge - const skillOrigin = classifySkillSource(skill.location) + const skillOrigin = classifySkillSource(skill.location, Instance.directory) + // altimate_change start — count workspace skill use for the session's summary rows + // Only a bound turn's use counts: a stale `_workspace` folder in an unbound project is not the workspace's. + if (skillOrigin === "workspace" && ctx.workspaceId !== undefined) { + Telemetry.noteWorkspaceUse(ctx.sessionID, "skills", 1, "add", ctx.turn) + } + // altimate_change end // altimate_change end // altimate_change start — telemetry instrumentation for skill loading with trigger classification @@ -265,6 +284,7 @@ export const SkillTool = Tool.define("skill", async (ctx) => { type: "skill_used", timestamp: Date.now(), session_id: ctx.sessionID, + workspace_id: ctx.workspaceId, // altimate_change — the turn's workspace message_id: ctx.messageID, skill_name: skill.name, skill_source: skillOrigin, diff --git a/packages/opencode/src/tool/tool.ts b/packages/opencode/src/tool/tool.ts index c7fa9f7c35..4558050b1d 100644 --- a/packages/opencode/src/tool/tool.ts +++ b/packages/opencode/src/tool/tool.ts @@ -59,6 +59,10 @@ export class InvalidArgumentsError extends Schema.TaggedErrorClass = { sessionID: SessionID messageID: MessageID + // altimate_change start — the calling turn's workspace and token, named on the rows a tool emits + workspaceId?: number + turn?: number + // altimate_change end agent: string abort: AbortSignal callID?: string @@ -170,6 +174,7 @@ function altimateTrackError(id: string, args: unknown, ctx: Context, startTime: type: "tool_call", timestamp: Date.now(), session_id: ctx.sessionID, + workspace_id: ctx.workspaceId, // altimate_change — the turn's workspace message_id: ctx.messageID, tool_name: id, tool_type: "standard", @@ -185,6 +190,7 @@ function altimateTrackError(id: string, args: unknown, ctx: Context, startTime: type: "core_failure", timestamp: Date.now(), session_id: ctx.sessionID, + workspace_id: ctx.workspaceId, // altimate_change — the turn's workspace tool_name: id, tool_category: Telemetry.categorizeToolName(id, "standard"), error_class: Telemetry.classifyError(errorMsg), @@ -215,6 +221,7 @@ function altimateTrackSuccess( type: "tool_call", timestamp: Date.now(), session_id: ctx.sessionID, + workspace_id: ctx.workspaceId, // altimate_change — the turn's workspace message_id: ctx.messageID, tool_name: id, tool_type: "standard", @@ -252,6 +259,7 @@ function altimateTrackSuccess( type: "core_failure", timestamp: Date.now(), session_id: ctx.sessionID, + workspace_id: ctx.workspaceId, // altimate_change — the turn's workspace tool_name: id, tool_category: Telemetry.categorizeToolName(id, "standard"), error_class: Telemetry.classifyError(errorMsg), @@ -269,6 +277,7 @@ function altimateTrackSuccess( type: "sql_quality", timestamp: Date.now(), session_id: ctx.sessionID, + workspace_id: ctx.workspaceId, // altimate_change — the turn's workspace tool_name: id, tool_category: toolCategory, finding_count: findings.length, diff --git a/packages/opencode/test/altimate/altimate-plugin.test.ts b/packages/opencode/test/altimate/altimate-plugin.test.ts index c6dbb791ce..3993fce4f2 100644 --- a/packages/opencode/test/altimate/altimate-plugin.test.ts +++ b/packages/opencode/test/altimate/altimate-plugin.test.ts @@ -5,6 +5,8 @@ import path from "path" import { buildAuthorizeUrl, buildCliContext } from "../../src/altimate/plugin/altimate" import { getOrCreateMachineId } from "../../src/altimate/util/machine-id" import { Config } from "../../src/config/config" +import { Telemetry } from "../../src/altimate/telemetry" +import os from "os" import { tmpdir } from "../fixture/fixture" const VALID_UUID = "550e8400-e29b-41d4-a716-446655440000" @@ -112,7 +114,7 @@ describe("buildCliContext", () => { expect(Object.prototype.hasOwnProperty.call(ctx, "machine_id")).toBe(false) }) - test("omits machine_id when Config.get() throws (fails CLOSED in the worker)", async () => { + test("omits machine_id when Config.get() throws and telemetry is off", async () => { delete process.env.ALTIMATE_TELEMETRY_DISABLED cfg.mockImplementation((async () => { throw new Error("InstanceRef not provided") @@ -120,14 +122,150 @@ describe("buildCliContext", () => { await using dir = await tmpdir() const idPath = path.join(dir.path, "machine-id") fs.writeFileSync(idPath, VALID_UUID, "utf8") + await Telemetry.shutdown() // under the runner the production sink is withheld: telemetry stays off const encoded = await buildCliContext(idPath) const ctx = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")) as Record - // Config unreadable → must NOT transmit the durable id. + // Config unreadable → the context follows the telemetry module, which is off here. + expect(Telemetry.isEnabled()).toBe(false) expect(Object.prototype.hasOwnProperty.call(ctx, "machine_id")).toBe(false) }) + test("includes machine_id when Config.get() throws but telemetry is on", async () => { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + cfg.mockImplementation((async () => { + throw new Error("InstanceRef not provided") + }) as never) + await using dir = await tmpdir() + const idPath = path.join(dir.path, "machine-id") + fs.writeFileSync(idPath, VALID_UUID, "utf8") + const savedCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + const savedOpencodeOptOut = process.env.OPENCODE_DISABLE_TELEMETRY + delete process.env.OPENCODE_DISABLE_TELEMETRY + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://ingest.example.invalid/" + const home = spyOn(os, "homedir").mockImplementation(() => dir.path) + const net = spyOn(globalThis, "fetch").mockImplementation((async () => new Response("", { status: 200 })) as any) + try { + await Telemetry.shutdown() + await Telemetry.init() + expect(Telemetry.isEnabled()).toBe(true) + + const encoded = await buildCliContext(idPath) + const ctx = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")) as Record + expect(ctx["machine_id"]).toBe(VALID_UUID) + } finally { + await Telemetry.shutdown() + net.mockRestore() + home.mockRestore() + if (savedCs === undefined) delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + else process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = savedCs + if (savedOpencodeOptOut === undefined) delete process.env.OPENCODE_DISABLE_TELEMETRY + else process.env.OPENCODE_DISABLE_TELEMETRY = savedOpencodeOptOut + } + }) + + // Config.get() throws (no Instance) while telemetry is ON for the process: the context reads the + // opt-out from the config files itself, and fails closed when it cannot read them. + describe.each([ + ["a config opt-out", JSON.stringify({ telemetry: { disabled: true } })], + ["an unreadable config", "{ not json"], + ])("Config.get() throws and the files hold %s", (_label, content) => { + test("omits machine_id even though telemetry is on", async () => { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + cfg.mockImplementation((async () => { + throw new Error("InstanceRef not provided") + }) as never) + await using dir = await tmpdir() + const idPath = path.join(dir.path, "machine-id") + fs.writeFileSync(idPath, VALID_UUID, "utf8") + const saved = { + cs: process.env.APPLICATIONINSIGHTS_CONNECTION_STRING, + opencodeOptOut: process.env.OPENCODE_DISABLE_TELEMETRY, + content: process.env.OPENCODE_CONFIG_CONTENT, + } + delete process.env.OPENCODE_DISABLE_TELEMETRY + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://ingest.example.invalid/" + const home = spyOn(os, "homedir").mockImplementation(() => dir.path) + const net = spyOn(globalThis, "fetch").mockImplementation((async () => new Response("", { status: 200 })) as any) + try { + await Telemetry.shutdown() + await Telemetry.init() + expect(Telemetry.isEnabled()).toBe(true) + process.env.OPENCODE_CONFIG_CONTENT = content + + const encoded = await buildCliContext(idPath) + const ctx = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")) as Record + expect(Object.prototype.hasOwnProperty.call(ctx, "machine_id")).toBe(false) + expect(ctx["v"]).toBe(1) + } finally { + await Telemetry.shutdown() + net.mockRestore() + home.mockRestore() + for (const [key, value] of [ + ["APPLICATIONINSIGHTS_CONNECTION_STRING", saved.cs], + ["OPENCODE_DISABLE_TELEMETRY", saved.opencodeOptOut], + ["OPENCODE_CONFIG_CONTENT", saved.content], + ] as const) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + } + }) + }) + + test("Config.get() throws: the opt-out of the project the command acts on is read, not the cwd's", async () => { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + cfg.mockImplementation((async () => { + throw new Error("InstanceRef not provided") + }) as never) + await using dir = await tmpdir() + await using project = await tmpdir() + fs.mkdirSync(path.join(project.path, ".altimate-code")) + fs.writeFileSync( + path.join(project.path, ".altimate-code", "altimate-code.json"), + JSON.stringify({ telemetry: { disabled: true } }), + ) + const idPath = path.join(dir.path, "machine-id") + fs.writeFileSync(idPath, VALID_UUID, "utf8") + const saved = { + cs: process.env.APPLICATIONINSIGHTS_CONNECTION_STRING, + opencodeOptOut: process.env.OPENCODE_DISABLE_TELEMETRY, + content: process.env.OPENCODE_CONFIG_CONTENT, + } + delete process.env.OPENCODE_DISABLE_TELEMETRY + delete process.env.OPENCODE_CONFIG_CONTENT + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://ingest.example.invalid/" + const home = spyOn(os, "homedir").mockImplementation(() => dir.path) + const net = spyOn(globalThis, "fetch").mockImplementation((async () => new Response("", { status: 200 })) as any) + const decode = (encoded: string) => + JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")) as Record + try { + await Telemetry.shutdown() + await Telemetry.init() + expect(Telemetry.isEnabled()).toBe(true) + // The opted-out project: no id. + expect("machine_id" in decode(await buildCliContext(idPath, project.path))).toBe(false) + // The same process from a directory with no opt-out: the id is sent. + expect(decode(await buildCliContext(idPath, dir.path))["machine_id"]).toBe(VALID_UUID) + } finally { + await Telemetry.shutdown() + net.mockRestore() + home.mockRestore() + for (const [key, value] of [ + ["APPLICATIONINSIGHTS_CONNECTION_STRING", saved.cs], + ["OPENCODE_DISABLE_TELEMETRY", saved.opencodeOptOut], + ["OPENCODE_CONFIG_CONTENT", saved.content], + ] as const) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + } + }) + test("includes machine_id when telemetry is enabled", async () => { delete process.env.ALTIMATE_TELEMETRY_DISABLED await using dir = await tmpdir() diff --git a/packages/opencode/test/altimate/plugin/workspace.test.ts b/packages/opencode/test/altimate/plugin/workspace.test.ts index 9670a24113..fad6d04623 100644 --- a/packages/opencode/test/altimate/plugin/workspace.test.ts +++ b/packages/opencode/test/altimate/plugin/workspace.test.ts @@ -35,6 +35,7 @@ const { recordEngineSkip, awaitKvReady, canInstallWith, + confirmedCopy, engineOfferInternals, showEngineInstallOffer, linkDialogInternals, @@ -1249,3 +1250,40 @@ describe("/integrations", () => { expect(h.selects[1]!.options[0]!.description).toContain("No session has attached yet") }) }) + +describe("confirmedCopy", () => { + const cmd = "npm i -g @altimateai/altimate-code-engine" + test("true only when the read-back holds the command", async () => { + let held = "" + const clipboard = { + write: async (data: string) => { + held = data + }, + read: async () => ({ data: `${held}\n`, mime: "text/plain" }), + } + expect(await confirmedCopy(clipboard, cmd)).toBe(true) + }) + test("false when the writer silently did nothing", async () => { + const clipboard = { + write: async () => {}, + read: async () => ({ data: "something else", mime: "text/plain" }), + } + expect(await confirmedCopy(clipboard, cmd)).toBe(false) + }) + test("false when the clipboard throws or cannot be read", async () => { + expect( + await confirmedCopy( + { + write: async () => { + throw new Error("no backend") + }, + read: async () => ({ data: cmd, mime: "text/plain" }), + }, + cmd, + ), + ).toBe(false) + expect(await confirmedCopy({ write: async () => {}, read: undefined }, cmd)).toBe(false) + // No writer: the clipboard holding the command already is not a copy. + expect(await confirmedCopy({ write: undefined, read: async () => ({ data: cmd, mime: "text/plain" }) }, cmd)).toBe(false) + }) +}) diff --git a/packages/opencode/test/altimate/telemetry/id-properties.test.ts b/packages/opencode/test/altimate/telemetry/id-properties.test.ts new file mode 100644 index 0000000000..f0ee99f944 --- /dev/null +++ b/packages/opencode/test/altimate/telemetry/id-properties.test.ts @@ -0,0 +1,82 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" +import { mkdtempSync, rmSync } from "fs" +import os from "os" +import path from "path" +import { Telemetry } from "../../../src/altimate/telemetry" + +describe("numeric ids are envelope properties, not measurements", () => { + let home: string + let posts: string[] + let fetchSpy: ReturnType + let homedirSpy: ReturnType + let savedCs: string | undefined + // Either disable flag wins over the connection string, so a developer or CI environment that + // sets one would fail the envelope assertions; cleared here and restored after. + const DISABLE_FLAGS = ["ALTIMATE_TELEMETRY_DISABLED", "OPENCODE_DISABLE_TELEMETRY"] as const + let savedFlags: Record + + beforeEach(() => { + home = mkdtempSync(path.join(os.tmpdir(), "telemetry-ids-")) + savedCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + savedFlags = {} + for (const flag of DISABLE_FLAGS) { + savedFlags[flag] = process.env[flag] + delete process.env[flag] + } + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://ingest.example.invalid/" + homedirSpy = spyOn(os, "homedir").mockImplementation(() => home) + posts = [] + fetchSpy = spyOn(globalThis, "fetch").mockImplementation((async (input: any, init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url + if (url.includes("/v2/track")) posts.push(String(init?.body ?? "")) + return new Response("", { status: 200 }) + }) as any) + }) + + afterEach(async () => { + await Telemetry.shutdown() + fetchSpy.mockRestore() + homedirSpy.mockRestore() + if (savedCs === undefined) delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + else process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = savedCs + for (const flag of DISABLE_FLAGS) { + if (savedFlags[flag] === undefined) delete process.env[flag] + else process.env[flag] = savedFlags[flag] + } + rmSync(home, { recursive: true, force: true }) + }) + + test("workspace ids ride in properties as strings; quantities stay measurements", async () => { + await Telemetry.shutdown() + await Telemetry.init() + Telemetry.track({ + type: "workspace_switched", + timestamp: Date.now(), + session_id: "s", + to_workspace_id: 9, + from_workspace_id: 4, + how: "rebind_remote", + }) + Telemetry.track({ + type: "workspace_api_error", + timestamp: Date.now(), + session_id: "s", + endpoint: "POST /datamate-project-bindings/bind", + status_class: "4xx", + status: 404, + workspace_id: 42, + }) + await Telemetry.flush() + const envelopes = posts.flatMap((b) => JSON.parse(b) as any[]) + const switched = envelopes.find((e) => e.data.baseData.name === "workspace_switched").data.baseData + expect(switched.properties.to_workspace_id).toBe("9") + expect(switched.properties.from_workspace_id).toBe("4") + expect(switched.properties.client).toBe("altimate-code") + expect("to_workspace_id" in switched.measurements).toBe(false) + const apiError = envelopes.find((e) => e.data.baseData.name === "workspace_api_error").data.baseData + expect(apiError.properties.workspace_id).toBe("42") + expect(apiError.measurements.status).toBe(404) // a quantity, still a measurement + expect("workspace_id" in apiError.measurements).toBe(false) + }) +}) diff --git a/packages/opencode/test/altimate/telemetry/onboarding.test.ts b/packages/opencode/test/altimate/telemetry/onboarding.test.ts index fd252a6c02..266a394eac 100644 --- a/packages/opencode/test/altimate/telemetry/onboarding.test.ts +++ b/packages/opencode/test/altimate/telemetry/onboarding.test.ts @@ -215,6 +215,25 @@ describe("activation events", () => { expect((events.find((e) => e.type === "first_job_completed") as any).job).toBe("sample_duck_db") }) + test("the first job's row names the workspace the hook input carries", async () => { + const events = captureEvents() + const hooks = await plugin() + await startOnboarding(hooks, "skip") + + // The trigger site sets `workspaceId` from the turn that ran the tool; the hook never asks the + // session for its current turn (a replacement's, when a cancelled turn's tool returns late). + await runTool( + hooks, + { tool: "sample_setup", sessionID: SESSION, callID: "c3w", args: {}, workspaceId: 7 }, + { title: "", output: "", metadata: { success: true } }, + ) + await settle() + + const completed = events.find((e) => e.type === "first_job_completed") as any + expect(completed.workspace_id).toBe(7) + expect("workspace_id" in (events.find((e) => e.type === "activation_job_selected") as any)).toBe(false) + }) + test("a failed sample setup counts as selected but not completed", async () => { const events = captureEvents() const hooks = await plugin() diff --git a/packages/opencode/test/altimate/telemetry/starter-action-picked.test.ts b/packages/opencode/test/altimate/telemetry/starter-action-picked.test.ts new file mode 100644 index 0000000000..6ce17d54ae --- /dev/null +++ b/packages/opencode/test/altimate/telemetry/starter-action-picked.test.ts @@ -0,0 +1,133 @@ +// altimate_change - new file +// +// `starter_action_picked`: one row per selection from a TUI starter dialog, naming the action, the +// prompt's position for a prompt, and the starter's own workspace; never the prompt's text. +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" +import * as Contents from "../../../src/altimate/workspace/contents" +import type { WorkspaceContents } from "../../../src/altimate/workspace/contents" +import { starterFor } from "../../../src/altimate/workspace/starter" +import { + starterActionPickedEvent, + starterPick, + trackStarterActionPicked, +} from "../../../src/altimate/workspace/starter-telemetry" +import { Telemetry } from "../../../src/altimate/telemetry" + +const contents: WorkspaceContents = { + skills: [{ name: "incident-runbook", description: "" }], + integrations: [], + memoryEnabled: true, + knowledge: { kind: "off" }, +} + +let events: any[] +let trackSpy: ReturnType + +beforeEach(() => { + events = [] + trackSpy = spyOn(Telemetry, "track").mockImplementation((e) => { + events.push(e) + }) +}) + +afterEach(() => { + trackSpy.mockRestore() +}) + +describe("starterPick", () => { + test.each([ + ["prompt:0", { action: "prompt", promptIndex: 0 }], + ["prompt:2", { action: "prompt", promptIndex: 2 }], + ["open", { action: "open" }], + ["done", { action: "done" }], + ] as const)("%s is a starter action", (value, pick) => { + expect(starterPick(value)).toEqual(pick) + }) + + test.each(["", "prompt:", "prompt:-1", "prompt:1.5", "prompt:x", "status", "Done", " done"])( + "%p is not a starter action", + (value) => { + expect(starterPick(value)).toBeNull() + }, + ) +}) + +describe("starter_action_picked row", () => { + test("a prompt carries its index and the workspace, and no text", () => { + const row = starterActionPickedEvent(7, { action: "prompt", promptIndex: 1 }, "ses_1") as Record + expect(Object.keys(row).sort()).toEqual( + ["action", "prompt_index", "session_id", "timestamp", "type", "workspace_id"].sort(), + ) + expect(row).toEqual( + expect.objectContaining({ + type: "starter_action_picked", + session_id: "ses_1", + workspace_id: 7, + action: "prompt", + prompt_index: 1, + }), + ) + }) + + test.each(["open", "done"] as const)("%s carries no prompt_index", (action) => { + const row = starterActionPickedEvent(7, { action }, "ses_1") as Record + expect(row.action).toBe(action) + expect("prompt_index" in row).toBe(false) + }) + + test("falls back to the process context's session", () => { + const row = starterActionPickedEvent(7, { action: "done" }) + expect(row).toEqual(expect.objectContaining({ session_id: Telemetry.getContext().sessionId })) + }) +}) + +describe("trackStarterActionPicked", () => { + test("tracks exactly one row per selection, against the starter's workspace", () => { + expect(trackStarterActionPicked({ workspaceId: 9 }, "prompt:2")).toBe(true) + expect(events).toHaveLength(1) + expect(events[0]).toEqual( + expect.objectContaining({ type: "starter_action_picked", workspace_id: 9, action: "prompt", prompt_index: 2 }), + ) + expect(JSON.stringify(events[0])).not.toContain("Which") + }) + + test("nothing before the starter arrives: the dialog is then the bare link confirmation", () => { + expect(trackStarterActionPicked(null, "done")).toBe(false) + expect(trackStarterActionPicked(null, "open")).toBe(false) + expect(events).toEqual([]) + }) + + test("nothing for a value that is not a starter action", () => { + expect(trackStarterActionPicked({ workspaceId: 9 }, "status")).toBe(false) + expect(events).toEqual([]) + }) + + test.each([undefined, 0, -3, 1.5, Number.NaN])("nothing for workspace id %p", (workspaceId) => { + expect(trackStarterActionPicked({ workspaceId }, "done")).toBe(false) + expect(events).toEqual([]) + }) + + test("never throws when telemetry does", () => { + trackSpy.mockImplementation(() => { + throw new Error("boom") + }) + expect(trackStarterActionPicked({ workspaceId: 9 }, "done")).toBe(false) + }) +}) + +describe("starterFor names its workspace", () => { + test("the served starter carries the id its starter_shown row reported", async () => { + const contentsSpy = spyOn(Contents, "workspaceContents").mockResolvedValue(contents) + try { + const starter = await starterFor("/tmp/project", { datamateId: 31, datamateName: "analytics" }) + expect(starter.workspaceId).toBe(31) + const shown = events.find((e) => e.type === "starter_shown") + expect(shown?.workspace_id).toBe(starter.workspaceId) + events.length = 0 + trackStarterActionPicked(starter, "prompt:0") + expect(events[0].workspace_id).toBe(shown.workspace_id) + } finally { + contentsSpy.mockRestore() + } + }) +}) diff --git a/packages/opencode/test/altimate/telemetry/workspace-context.test.ts b/packages/opencode/test/altimate/telemetry/workspace-context.test.ts new file mode 100644 index 0000000000..85dc8ff8e0 --- /dev/null +++ b/packages/opencode/test/altimate/telemetry/workspace-context.test.ts @@ -0,0 +1,199 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" +import { mkdtempSync, rmSync } from "fs" +import os from "os" +import path from "path" +import { Telemetry } from "../../../src/altimate/telemetry" + +describe("workspace id on the telemetry context", () => { + let home: string + let posts: string[] + let fetchSpy: ReturnType + let homedirSpy: ReturnType + let savedCs: string | undefined + let savedHome: string | undefined + // Either disable flag wins over the connection string; cleared for the real init below and restored. + const DISABLE_FLAGS = ["ALTIMATE_TELEMETRY_DISABLED", "OPENCODE_DISABLE_TELEMETRY"] as const + let savedFlags: Record + + beforeEach(() => { + home = mkdtempSync(path.join(os.tmpdir(), "telemetry-ws-")) + savedCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + savedFlags = {} + for (const flag of DISABLE_FLAGS) { + savedFlags[flag] = process.env[flag] + delete process.env[flag] + } + savedHome = process.env.OPENCODE_TEST_HOME + process.env.OPENCODE_TEST_HOME = home + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://ingest.example.invalid/" + homedirSpy = spyOn(os, "homedir").mockImplementation(() => home) + posts = [] + fetchSpy = spyOn(globalThis, "fetch").mockImplementation((async (input: any, init?: RequestInit) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url + if (url.includes("/v2/track")) posts.push(String(init?.body ?? "")) + return new Response("", { status: 200 }) + }) as any) + }) + + afterEach(async () => { + await Telemetry.shutdown() + fetchSpy.mockRestore() + homedirSpy.mockRestore() + if (savedCs === undefined) delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + else process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = savedCs + if (savedHome === undefined) delete process.env.OPENCODE_TEST_HOME + else process.env.OPENCODE_TEST_HOME = savedHome + for (const flag of DISABLE_FLAGS) { + if (savedFlags[flag] === undefined) delete process.env[flag] + else process.env[flag] = savedFlags[flag] + } + rmSync(home, { recursive: true, force: true }) + }) + + const envelopes = () => posts.flatMap((b) => JSON.parse(b) as any[]) + const startup = (command: string) => + ({ + type: "startup_ready", + session_id: "", + command, + surface: command, + duration_ms: 1, + fresh_install: false, + timestamp: Date.now(), + }) as const + + test("workspace use is counted per session and cleared explicitly", () => { + Telemetry.clearWorkspaceUse("s9") + expect(Telemetry.workspaceUseOf("s9")).toEqual({ skills: 0, memory: 0, integrations: 0 }) + Telemetry.noteWorkspaceUse("s9", "skills") + Telemetry.noteWorkspaceUse("s9", "memory", 3) + Telemetry.noteWorkspaceUse("s9", "integrations") + Telemetry.noteWorkspaceUse("s9", "integrations") + Telemetry.noteWorkspaceUse("", "skills") // no session: ignored + Telemetry.noteWorkspaceUse("s9", "skills", 0) // nothing to count + expect(Telemetry.workspaceUseOf("s9")).toEqual({ skills: 1, memory: 3, integrations: 2 }) + // Memory is injected on every step of a turn: "max" keeps the largest injection, not a sum. + Telemetry.noteWorkspaceUse("s9", "memory", 2, "max") + Telemetry.noteWorkspaceUse("s9", "memory", 5, "max") + Telemetry.noteWorkspaceUse("s9", "memory", 4, "max") + expect(Telemetry.workspaceUseOf("s9").memory).toBe(5) + expect(Telemetry.workspaceUseOf("other")).toEqual({ skills: 0, memory: 0, integrations: 0 }) + Telemetry.clearWorkspaceUse("s9") + expect(Telemetry.workspaceUseOf("s9")).toEqual({ skills: 0, memory: 0, integrations: 0 }) + }) + + test("a turn ends only its own counters; a stale turn's use does not count", () => { + const first = Telemetry.setContext({ sessionId: "s7", projectId: "p1" }) + Telemetry.noteWorkspaceUse("s7", "skills") + const second = Telemetry.setContext({ sessionId: "s7", projectId: "p1" }) + expect(second).toBeGreaterThan(first) + Telemetry.noteWorkspaceUse("s7", "memory", 2) + // Counts are per turn: the cancelled turn's rows read its own, the replacement's its own. + expect(Telemetry.workspaceUseOf("s7", first)).toEqual({ skills: 1, memory: 0, integrations: 0 }) + expect(Telemetry.workspaceUseOf("s7", second)).toEqual({ skills: 0, memory: 2, integrations: 0 }) + Telemetry.endSession("s7", first) // the old generation, after the replacement started + expect(Telemetry.workspaceUseOf("s7", first)).toEqual({ skills: 0, memory: 0, integrations: 0 }) + expect(Telemetry.workspaceUseOf("s7").memory).toBe(2) + // A use landing with a stale turn does not count toward the replacement. + Telemetry.noteWorkspaceUse("s7", "integrations", 1, "add", first) + expect(Telemetry.workspaceUseOf("s7").integrations).toBe(0) + Telemetry.noteWorkspaceUse("s7", "integrations", 1, "add", second) + expect(Telemetry.workspaceUseOf("s7").integrations).toBe(1) + Telemetry.endSession("s7", second) + expect(Telemetry.workspaceUseOf("s7")).toEqual({ skills: 0, memory: 0, integrations: 0 }) + // Tokens never repeat, so a stale end can never hit a later turn. + const stale = Telemetry.setContext({ sessionId: "s8", projectId: "p1" }) + Telemetry.endSession("s8", stale) + const fresh = Telemetry.setContext({ sessionId: "s8", projectId: "p1" }) + expect(fresh).toBeGreaterThan(stale) + Telemetry.noteWorkspaceUse("s8", "skills") + Telemetry.endSession("s8", stale) + expect(Telemetry.workspaceUseOf("s8", fresh).skills).toBe(1) + Telemetry.endSession("s8", fresh) + Telemetry.setContext({ sessionId: "", projectId: "" }) + }) + + test("a row carries the workspace it names and nothing is stamped from the context", async () => { + await Telemetry.shutdown() + await Telemetry.init() + Telemetry.setContext({ sessionId: "s1", projectId: "p1" }) + // The context holds no workspace: a row names its own. + expect("workspaceId" in Telemetry.getContext()).toBe(false) + + Telemetry.track(startup("bound")) // names none: carries none, whatever the context holds + Telemetry.track({ + type: "workspace_linked", + timestamp: Date.now(), + session_id: "s1", + workspace_id: 7, + how: "bind_existing", + }) + Telemetry.track({ type: "workspace_unlinked", timestamp: Date.now(), session_id: "s1" }) + Telemetry.track({ type: "workspace_unlinked", timestamp: Date.now(), session_id: "s1", workspace_id: undefined }) + await Telemetry.flush() + const bound = envelopes().find((e) => e.data.baseData.properties.command === "bound") + expect("workspace_id" in bound.data.baseData.properties).toBe(false) + expect(bound.data.baseData.properties.client).toBe("altimate-code") + const linked = envelopes().find((e) => e.data.baseData.name === "workspace_linked") + expect(linked.data.baseData.properties.workspace_id).toBe("7") + const unlinked = envelopes().filter((e) => e.data.baseData.name === "workspace_unlinked") + expect(unlinked.map((e) => e.data.baseData.properties.workspace_id)).toEqual([undefined, undefined]) + Telemetry.endSession("s1") + }) + + // Every event type the workspace rows added, one each. + const workspaceRows = (): Telemetry.Event[] => { + const base = { timestamp: Date.now(), session_id: "s1" } + return [ + { ...base, type: "workspace_linked", workspace_id: 1, how: "bind_existing" }, + { ...base, type: "workspace_created", workspace_id: 1, how: "create_and_bind" }, + { ...base, type: "workspace_link_failed", how: "bind_existing", reason: "auth" }, + { ...base, type: "workspace_unlinked", workspace_id: 1 }, + { ...base, type: "workspace_switched", to_workspace_id: 2, how: "rebind_remote" }, + { ...base, type: "workspace_auth_expired", endpoint: "GET /datamates/", status: 401 }, + { ...base, type: "workspace_api_error", endpoint: "GET /datamates/", status_class: "5xx", status: 500 }, + { ...base, type: "starter_shown", workspace_id: 1, knowledge: "off" }, + { ...base, type: "starter_action_picked", workspace_id: 1, action: "done" }, + { ...base, type: "workspace_memory_contributed", workspace_id: 1, block_count: 1 }, + { ...base, type: "workspace_sync_failed", workspace_id: 1, kind: "skills", reason: "network" }, + { ...base, type: "workspace_engine_attach", outcome: "attached", workspace_id: 1 }, + { ...base, type: "workspace_engine_install_prompt", action: "shown", reason: "engine-missing" }, + { + ...base, + type: "workspace_engine_precedence", + enabled: true, + shadowed_types: 0, + shadowed_connections: 0, + extensions: 0, + escape_hatch: false, + }, + { ...base, type: "workspace_engine_redirect", tool: "datamate", operation: "status", outcome: "refused" }, + { ...base, type: "workspace_engine_error", stage: "attach", message: "boom" }, + ] + } + + test("the workspace rows are sent when telemetry is on and dropped when it is off", async () => { + const names = workspaceRows().map((e) => e.type) + expect(new Set(names).size).toBe(16) // plus tool_call rows, which are not new types + + await Telemetry.shutdown() + await Telemetry.init() + for (const row of workspaceRows()) Telemetry.track(row) + await Telemetry.flush() + const sent = new Set(envelopes().map((e) => e.data.baseData.name)) + for (const name of names) expect(sent.has(name)).toBe(true) + + for (const flag of DISABLE_FLAGS) { + posts = [] + await Telemetry.shutdown() + process.env[flag] = "1" + await Telemetry.init() + expect(Telemetry.isEnabled()).toBe(false) + for (const row of workspaceRows()) Telemetry.track(row) + await Telemetry.flush() + expect(posts).toEqual([]) + delete process.env[flag] + } + }) +}) diff --git a/packages/opencode/test/altimate/training-store-session.test.ts b/packages/opencode/test/altimate/training-store-session.test.ts new file mode 100644 index 0000000000..feb1019397 --- /dev/null +++ b/packages/opencode/test/altimate/training-store-session.test.ts @@ -0,0 +1,25 @@ +import { afterEach, describe, expect, spyOn, test } from "bun:test" +import { MemoryStore } from "../../src/memory" +import { TrainingStore } from "../../src/altimate/training" + +describe("TrainingStore.save", () => { + const spies: Array> = [] + afterEach(() => { + for (const s of spies.splice(0)) s.mockRestore() + }) + + test("passes the writing session to the memory write, for the workspace write-back's row", async () => { + spies.push(spyOn(MemoryStore, "read").mockResolvedValue(undefined as any)) + const write = spyOn(MemoryStore, "write").mockResolvedValue({ duplicates: [] }) + spies.push(write) + await TrainingStore.save({ + kind: "rule", + name: "naming", + scope: "project", + content: "Use snake_case.", + sessionID: "s-7", + }) + expect(write).toHaveBeenCalledTimes(1) + expect(write.mock.calls[0][1]).toEqual({ sessionID: "s-7" }) + }) +}) diff --git a/packages/opencode/test/altimate/workspace/browser-handoff.test.ts b/packages/opencode/test/altimate/workspace/browser-handoff.test.ts index c3df0d561a..d5dc0314e8 100644 --- a/packages/opencode/test/altimate/workspace/browser-handoff.test.ts +++ b/packages/opencode/test/altimate/workspace/browser-handoff.test.ts @@ -7,10 +7,11 @@ // launching a real browser. The listener itself binds to 127.0.0.1, walks // 7317..7325, and processes real HTTP requests — this is genuine end-to-end // coverage for the callback validation path. -import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" import { createServer, connect } from "node:net" import { AltimateApi } from "../../../src/altimate/api/client" +import * as AltimatePlugin from "../../../src/altimate/plugin/altimate" import { buildManageUrl, openWorkspaceBrowserHandoff, @@ -344,6 +345,25 @@ describe("runHandoffWithOpener end-to-end", () => { } }) + test("cli_context is built for the linked project's directory, so its config opt-out applies", async () => { + const spy = spyOn(AltimatePlugin, "buildCliContext").mockImplementation(async () => "ctx") + try { + let observed = "" + await runHandoffWithOpener( + { identifier: { repoRemote: "git@github.com:acme/foo.git", projectPath: "/w/foo" }, projectName: "foo" }, + async (url) => { + observed = url + const { state, redirect } = parseHandoffUrl(url) + await fireCallback(redirect, { workspace_id: "1", state, tenant: "acme" }) + }, + ) + expect(spy).toHaveBeenCalledWith(undefined, "/w/foo") + expect(new URLSearchParams(new URL(observed).hash.replace(/^#/, "")).get("cli_context")).toBe("ctx") + } finally { + spy.mockRestore() + } + }) + test("URL keeps project_name, project_remote, project_path out of the query", async () => { // Per m6 in the consensus review: project_path + project_remote MUST NOT // be sent as query params (they'd land in browser history, SaaS/CDN/WAF diff --git a/packages/opencode/test/altimate/workspace/datamate-manager-gate.test.ts b/packages/opencode/test/altimate/workspace/datamate-manager-gate.test.ts index 4db9c869f9..301b511517 100644 --- a/packages/opencode/test/altimate/workspace/datamate-manager-gate.test.ts +++ b/packages/opencode/test/altimate/workspace/datamate-manager-gate.test.ts @@ -38,8 +38,10 @@ import { DATAMATE_MANAGER_TOOL_ID, engineNotice, hiddenToolIds, + refusal, turnNotice, } from "../../../src/altimate/workspace/datamate-manager-gate" +import { Telemetry } from "../../../src/altimate/telemetry" const ORIGINAL_FLAG = process.env.ALTIMATE_DISABLE_WORKSPACE const ORIGINAL_SPEC = process.env.ALTIMATE_ENGINE_INSTALL_SPEC @@ -815,3 +817,28 @@ describe("older datamate- entries in a linked project", () => { }) } }) + +describe("telemetry — the refusal", () => { + test("a refused datamate_manager call is reported as a redirect naming the operation and workspace", () => { + const rows: Array> = [] + const spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e as Record) + }) + try { + const out = refusal("status", { id: "42", name: "analytics" }, "ses_r") + expect(out.metadata).toEqual({ operation: "status", managedBy: "42" }) + expect(rows).toEqual([ + expect.objectContaining({ + type: "workspace_engine_redirect", + tool: "datamate", + operation: "status", + outcome: "refused", + workspace_id: 42, + session_id: "ses_r", + }), + ]) + } finally { + spy.mockRestore() + } + }) +}) diff --git a/packages/opencode/test/altimate/workspace/engine-install-offer.test.ts b/packages/opencode/test/altimate/workspace/engine-install-offer.test.ts index d4b75772cd..d99aaf2be4 100644 --- a/packages/opencode/test/altimate/workspace/engine-install-offer.test.ts +++ b/packages/opencode/test/altimate/workspace/engine-install-offer.test.ts @@ -22,8 +22,14 @@ import { type EngineOffer, type Toast, } from "../../../src/altimate/workspace/engine-overlay" -import { OFFER_RECHECK_MS, OFFER_SKIP_TTL_MS } from "../../../src/altimate/workspace/engine-offer" +import { + OFFER_RECHECK_MS, + OFFER_SKIP_TTL_MS, + decodeOfferDetail, + trackPrintedOffer, +} from "../../../src/altimate/workspace/engine-offer" import type { CachedBinding } from "../../../src/altimate/workspace/state" +import { Telemetry } from "../../../src/altimate/telemetry" const DIR = "/tmp/analytics" const ORIGINAL_FLAG = process.env.ALTIMATE_DISABLE_WORKSPACE @@ -37,7 +43,14 @@ const binding: CachedBinding = { linkedAt: 0, } as CachedBinding -type Harness = { offers: EngineOffer[]; toasts: Toast[]; printed: string[]; published: number; publishedFor: string[] } +type Harness = { + offers: EngineOffer[] + toasts: Toast[] + printed: string[] + published: number + publishedFor: string[] + publishedDetail: string[] +} /** No engine on PATH (or an old one) plus captured surfaces. */ function install(opts: { @@ -49,7 +62,7 @@ function install(opts: { surface?: boolean bound?: boolean }): Harness { - const h: Harness = { offers: [], toasts: [], printed: [], published: 0, publishedFor: [] } + const h: Harness = { offers: [], toasts: [], printed: [], published: 0, publishedFor: [], publishedDetail: [] } delete process.env.ALTIMATE_DISABLE_WORKSPACE syncInternals.serve = () => false syncInternals.headless = () => opts.headless === true @@ -67,10 +80,11 @@ function install(opts: { syncInternals.printLine = (line) => { h.printed.push(line) } - syncInternals.publishOffer = async (sessionID) => { + syncInternals.publishOffer = async (sessionID, detail) => { if (opts.bus === false) return false h.published += 1 h.publishedFor.push(sessionID) + h.publishedDetail.push(detail) return true } if (opts.surface) { @@ -160,6 +174,43 @@ describe("offer routing — engine missing", () => { expect(h.toasts).toEqual([]) expect(h.printed).toEqual([]) }) + test("the published offer carries its fields, and the surface that prints them reports shown", async () => { + const h = install({}) + await beforeTurn("s1") + expect(decodeOfferDetail(h.publishedDetail[0])).toEqual({ + reason: "engine-missing", + workspaceId: "42", + declared: 2, + found: undefined, + }) + const rows: Record[] = [] + const spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e as Record) + }) + try { + // An attached run prints the offer from this event in its own process and reports from it. + expect(trackPrintedOffer(h.publishedDetail[0], "s1")).toBe(true) + expect(trackPrintedOffer(undefined, "s1")).toBe(false) + expect(trackPrintedOffer("not json", "s1")).toBe(false) + expect(trackPrintedOffer(JSON.stringify({ reason: "other", workspaceId: "42" }), "s1")).toBe(false) + expect(trackPrintedOffer(JSON.stringify({ reason: "engine-missing", declared: "2" }), "s1")).toBe(false) + } finally { + spy.mockRestore() + } + expect(rows).toEqual([ + expect.objectContaining({ + type: "workspace_engine_install_prompt", + action: "shown", + surface: "bus", + session_id: "s1", + workspace_id: 42, + reason: "engine-missing", + declared: 2, + }), + ]) + // No node fields: the probe stays off the dispatch path, so the event carries none. + expect("node_detected" in rows[0]!).toBe(false) + }) test("falls back to the toast only when the bus is unavailable", async () => { const h = install({ bus: false }) await beforeTurn("s1") @@ -460,3 +511,57 @@ describe("install success is verified, not assumed", () => { expect(await installEngine()).toEqual({ ok: false, error: "boom EACCES denied" }) }) }) + +describe("telemetry — the install prompt", () => { + const rows: Array> = [] + let spy: ReturnType + beforeEach(() => { + rows.length = 0 + spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e as Record) + }) + }) + afterEach(() => spy.mockRestore()) + const settle = () => new Promise((resolve) => setTimeout(resolve, 0)) + const prompts = () => rows.filter((r) => r.type === "workspace_engine_install_prompt") + + test("headless: shown once per session, naming the surface and the node that could install", async () => { + install({ headless: true }) + syncInternals.nodeMajor = async () => 22 + await beforeTurn("s1") + await beforeTurn("s1") + await settle() + expect(prompts()).toEqual([ + expect.objectContaining({ + session_id: "s1", + action: "shown", + reason: "engine-missing", + workspace_id: 42, + declared: 2, + surface: "headless", + node_detected: true, + node_major: 22, + }), + ]) + // The missing engine is the attach outcome behind the prompt, reported once as well. + expect(rows.filter((r) => r.type === "workspace_engine_attach")).toEqual([ + expect.objectContaining({ outcome: "engine_missing", declared: 2 }), + ]) + }) + + test("a dialog reports shown itself when it renders, so dispatch over the bus reports nothing here", async () => { + install({}) + await beforeTurn("s1") + await settle() + expect(prompts()).toHaveLength(0) + }) + + test("the toast reports shown once shown, and no node on PATH is said rather than guessed", async () => { + install({ bus: false }) + await beforeTurn("s1") + await settle() + expect(prompts()).toHaveLength(1) + expect(prompts()[0]).toEqual(expect.objectContaining({ surface: "toast", node_detected: false })) + expect("node_major" in prompts()[0]).toBe(false) + }) +}) diff --git a/packages/opencode/test/altimate/workspace/engine-overlay.test.ts b/packages/opencode/test/altimate/workspace/engine-overlay.test.ts index ca845e28f5..601a853366 100644 --- a/packages/opencode/test/altimate/workspace/engine-overlay.test.ts +++ b/packages/opencode/test/altimate/workspace/engine-overlay.test.ts @@ -29,6 +29,7 @@ import { } from "../../../src/altimate/workspace/engine-overlay" import type { ScopedBinding } from "../../../src/altimate/workspace/engine-seams" import type { AttachSnapshot } from "../../../src/altimate/workspace/attach-snapshot" +import { Telemetry } from "../../../src/altimate/telemetry" import { log } from "../../../src/altimate/workspace/engine-seams" import { DATAMATE_KEY } from "../../../src/altimate/datamate-transport" @@ -1262,3 +1263,100 @@ describe("beforeTurn — what a turn boundary does", () => { expect(managedWorkspace()).toEqual({ id: "42", name: "analytics" }) }) }) + +describe("telemetry — the attach", () => { + const rows: Array> = [] + let spy: ReturnType + beforeEach(() => { + rows.length = 0 + spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e as Record) + }) + }) + afterEach(() => spy.mockRestore()) + const attaches = () => rows.filter((r) => r.type === "workspace_engine_attach") + + test("a settled attach is reported once with its inventory; a repeated boundary adds nothing; a change reports again", async () => { + const report = [{ key: "dbt_execute_sql", integrationId: "dbt", reason: "invalid-connection" }] + const h = install({ meta: { [UNFULFILLED_META_KEY]: report } }) + await beforeTurn("s1") + await beforeTurn("s1") + expect(attaches()).toEqual([ + expect.objectContaining({ + session_id: "s1", + workspace_id: 42, + outcome: "attached", + available: 2, + declared: 3, + missing: 1, + engine_version: "0.7.3", + }), + ]) + h.binding = null + await beforeTurn("s1") + expect(attaches().map((r) => r.outcome)).toEqual(["attached", "unbound"]) + }) + + test("the retry sentinel is not a settled attach: a retry that succeeds reports one attach, one that fails reports one failure", async () => { + const h = install({ status: "failed" }) + h.onAdd = () => { + h.status = "connected" + } + await beforeTurn("s1") + await beforeTurn("s1") + expect(attaches().map((r) => r.outcome)).toEqual(["attached"]) + rows.length = 0 + install({ status: "failed", statusError: "Connection closed" }) + await beforeTurn("s2") + await beforeTurn("s2") + expect(attaches().map((r) => r.outcome)).toEqual(["connect_failed"]) + // A retry whose add throws settles as a failure too, rather than vanishing into the turn hook. + rows.length = 0 + const g = install({ status: "failed" }) + g.onAdd = () => { + throw new Error("spawn failed") + } + await beforeTurn("s3") + expect(settledOutcome("s3")?.kind).toBe("connect-failed") + expect(attaches().map((r) => r.outcome)).toEqual(["connect_failed"]) + }) + + test("a relink to another workspace reports the attach again, named for the new workspace", async () => { + const h = install({}) + await beforeTurn("s1") + h.binding = bound(7, "growth") + await beforeTurn("s1") + await beforeTurn("s1") + expect(attaches().map((r) => [r.outcome, r.workspace_id])).toEqual([ + ["attached", 42], + ["attached", 7], + ]) + // The same id under another account is another workspace: reported again. + h.binding = bound(7, "growth", "other|https://api.other.example") + await beforeTurn("s1") + expect(attaches().map((r) => [r.outcome, r.workspace_id])).toEqual([ + ["attached", 42], + ["attached", 7], + ["attached", 7], + ]) + }) + + test("an engine below the floor reports the version it found", async () => { + install({ version: "0.1.0" }) + await beforeTurn("s1") + expect(attaches()).toEqual([expect.objectContaining({ outcome: "engine_too_old", found_version: "0.1.0" })]) + }) + + test("a probe that throws during the overlay is an attach-stage error with a masked message", async () => { + const h = install({ mcp: { datamate: IDE_ENTRY } }) + syncInternals.versionOf = async () => { + throw new Error("probe exploded at /home/someone/bin/datamate") + } + await overlay(DIR, h.config) + const errors = rows.filter((r) => r.type === "workspace_engine_error") + expect(errors).toHaveLength(1) + expect(errors[0]).toEqual(expect.objectContaining({ stage: "attach" })) + expect(String(errors[0].message)).toContain("probe exploded") + expect(String(errors[0].message)).not.toContain("/home/someone") + }) +}) diff --git a/packages/opencode/test/altimate/workspace/engine-telemetry.test.ts b/packages/opencode/test/altimate/workspace/engine-telemetry.test.ts new file mode 100644 index 0000000000..4c0e84ce6b --- /dev/null +++ b/packages/opencode/test/altimate/workspace/engine-telemetry.test.ts @@ -0,0 +1,189 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" +import { Telemetry } from "../../../src/altimate/telemetry" +import { + attachEvent, + attachKey, + attachSignature, + connectErrorClass, + installPromptEvent, + precedenceEvent, + resetEngineTelemetryForTests, + trackAttach, + trackEngineError, + trackPrecedence, + trackRedirect, +} from "../../../src/altimate/workspace/engine-telemetry" + +describe("workspace engine telemetry builders", () => { + let events: any[] + let spy: ReturnType + beforeEach(() => { + events = [] + spy = spyOn(Telemetry, "track").mockImplementation((e) => { + events.push(e) + }) + resetEngineTelemetryForTests() + }) + afterEach(() => { + spy.mockRestore() + resetEngineTelemetryForTests() + }) + + test("attach outcomes map to the event with counts, versions and an error class", () => { + expect( + attachEvent({ kind: "attached", available: 3, declared: 4, missing: ["x"] }, { engineVersion: "1.2.3" }), + ).toEqual( + expect.objectContaining({ + type: "workspace_engine_attach", + outcome: "attached", + available: 3, + declared: 4, + missing: 1, + engine_version: "1.2.3", + }), + ) + expect(attachEvent({ kind: "engine-missing", declared: 2 })).toEqual( + expect.objectContaining({ outcome: "engine_missing", declared: 2 }), + ) + expect(attachEvent({ kind: "engine-too-old", found: "0.9.0" })).toEqual( + expect.objectContaining({ outcome: "engine_too_old", found_version: "0.9.0" }), + ) + expect(attachEvent({ kind: "connect-failed", error: "the workspace link could not be read" })).toEqual( + expect.objectContaining({ outcome: "connect_failed", error_class: "link_unreadable" }), + ) + expect(attachEvent({ kind: "unbound" })).toEqual(expect.objectContaining({ outcome: "unbound" })) + expect(connectErrorClass("the workspace engine could not be checked")).toBe("probe_failed") + expect((attachEvent({ kind: "unbound" }, {}, "ses_x") as any).session_id).toBe("ses_x") + expect(connectErrorClass("something else")).toBe("other") + }) + + test("an attach is tracked when the outcome changed, not on every turn boundary", () => { + const attached = { kind: "attached", available: 2 } as const + expect(trackAttach(attached, undefined)).toBe(true) + expect(trackAttach(attached, attachKey(attached))).toBe(false) + expect(trackAttach({ kind: "attached", available: 3 }, attachKey(attached))).toBe(true) + expect(trackAttach({ kind: "engine-missing" }, attachKey({ kind: "attached", available: 3 }))).toBe(true) + // The same-shaped attach to another workspace (a relink) is a change, and the identity is the + // account-scoped key: the same id under another account is another workspace. + expect( + trackAttach(attached, attachKey(attached, "acme|42"), { workspaceId: 42, workspaceIdentity: "acme|42" }), + ).toBe(false) + expect(trackAttach(attached, attachKey(attached, "acme|42"), { workspaceId: 7, workspaceIdentity: "acme|7" })).toBe( + true, + ) + expect( + trackAttach(attached, attachKey(attached, "acme|42"), { workspaceId: 42, workspaceIdentity: "other|42" }), + ).toBe(true) + expect(events.map((e) => e.outcome)).toEqual(["attached", "attached", "engine_missing", "attached", "attached"]) + expect(attachSignature({ kind: "connect-failed", error: "a" })).toBe( + attachSignature({ kind: "connect-failed", error: "b" }), + ) + expect(attachSignature({ kind: "attached", available: 2, extensions: [{ integration: "x" } as any] })).not.toBe( + attachSignature({ kind: "attached", available: 2 }), + ) + expect((attachEvent({ kind: "attached", available: 1 }, { workspaceId: 42 }) as any).workspace_id).toBe(42) + // An unbound attach names no workspace, explicitly: the key is present and undefined, so the + // session's entry cannot be stamped onto it. + const unbound = attachEvent({ kind: "unbound" }, { workspaceId: 42 }) as any + expect("workspace_id" in unbound).toBe(true) + expect(unbound.workspace_id).toBeUndefined() + }) + + test("the install prompt event carries the offer, the surface and node detection", () => { + const offer = { reason: "engine-missing" as const, workspaceId: "17", declared: 2 } + expect(installPromptEvent("shown", offer, { surface: "headless", nodeMajor: 22 })).toEqual( + expect.objectContaining({ + type: "workspace_engine_install_prompt", + action: "shown", + reason: "engine-missing", + workspace_id: 17, + declared: 2, + surface: "headless", + node_detected: true, + node_major: 22, + }), + ) + const shown = installPromptEvent( + "shown", + { reason: "engine-too-old", workspaceId: "x", found: "0.9.0" }, + { nodeMajor: null }, + ) as any + expect(shown.node_detected).toBe(false) + expect("node_major" in shown).toBe(false) + expect("workspace_id" in shown).toBe(false) + expect(shown.found_version).toBe("0.9.0") + expect(installPromptEvent("declined", offer)).toEqual( + expect.objectContaining({ action: "declined", workspace_id: 17 }), + ) + }) + + test("precedence snapshots are tracked per session when they change", () => { + const shadowed = new Map([ + [ + "snowflake", + new Map([ + ["sql_execute", {} as any], + ["sql_explain", {} as any], + ]), + ], + ["dbt", new Map([["dbt_run", {} as any]])], + ]) + const on: any = { enabled: true, shadowed, extensions: [{ integration: "x", tools: [] }], workspaceId: "7" } + // A disabled snapshot carries no binding; the directory's linked workspace names the row. + const offSnap: any = { enabled: false, disabledReason: "escape-hatch" as const, shadowed: new Map() } + expect( + (precedenceEvent(offSnap, { escapeHatch: true, connections: 0, types: 0, workspaceId: 7 }) as any).workspace_id, + ).toBe(7) + // Unbound and no linked workspace: the key is present and undefined, so the row names none. + const unboundRow = precedenceEvent(offSnap, { escapeHatch: true, connections: 0, types: 0 }) as any + expect("workspace_id" in unboundRow).toBe(true) + expect(unboundRow.workspace_id).toBeUndefined() + expect(precedenceEvent(on, { escapeHatch: false, connections: 1, types: 2 })).toEqual( + expect.objectContaining({ + type: "workspace_engine_precedence", + enabled: true, + shadowed_types: 2, + shadowed_connections: 1, + extensions: 1, + escape_hatch: false, + workspace_id: 7, + }), + ) + expect(trackPrecedence("s1", on, { escapeHatch: false, connections: 1, types: 2 })).toBe(true) + expect(trackPrecedence("s1", on, { escapeHatch: false, connections: 1, types: 2 })).toBe(false) // unchanged + expect(trackPrecedence("s1", on, { escapeHatch: false, connections: 2, types: 2 })).toBe(true) // a connection was added locally + expect(trackPrecedence("s2", on, { escapeHatch: false, connections: 2, types: 2 })).toBe(true) // another session + const off: any = { enabled: false, disabledReason: "escape-hatch" as const, shadowed: new Map(), workspaceId: "7" } + expect(trackPrecedence("s1", off, { escapeHatch: true, connections: 0, types: 0 })).toBe(true) + expect(events.at(-1)).toEqual( + expect.objectContaining({ enabled: false, reason: "escape-hatch", shadowed_types: 0, escape_hatch: true }), + ) + expect(trackPrecedence("s1", off, { escapeHatch: false, connections: 0, types: 0 })).toBe(true) // the hatch toggled + // The same id under another account is another workspace: reported again. + expect(trackPrecedence("s3", on, { escapeHatch: false, connections: 1, types: 2, identity: "acme|7" })).toBe(true) + expect(trackPrecedence("s3", on, { escapeHatch: false, connections: 1, types: 2, identity: "acme|7" })).toBe(false) + expect(trackPrecedence("s3", on, { escapeHatch: false, connections: 1, types: 2, identity: "other|7" })).toBe(true) + expect((events.at(-1) as any).escape_hatch).toBe(false) + }) + + test("errors are masked and bounded; a refusal names the operation and workspace", () => { + trackEngineError("attach", "ENOENT: /home/someone/.altimate/engine " + "x".repeat(500)) + const err = events.at(-1) + expect(err.type).toBe("workspace_engine_error") + expect(err.stage).toBe("attach") + expect(err.message.length).toBeLessThanOrEqual(200) + expect(err.message).not.toContain("/home/someone") + trackEngineError("precedence", "derive boom", "ses_p") + expect(events.at(-1)).toEqual(expect.objectContaining({ stage: "precedence", session_id: "ses_p" })) + trackRedirect("datamate", "status", 9) + expect(events.at(-1)).toEqual( + expect.objectContaining({ + type: "workspace_engine_redirect", + tool: "datamate", + operation: "status", + outcome: "refused", + workspace_id: 9, + }), + ) + }) +}) diff --git a/packages/opencode/test/altimate/workspace/link-telemetry.test.ts b/packages/opencode/test/altimate/workspace/link-telemetry.test.ts new file mode 100644 index 0000000000..9ff3d5ce29 --- /dev/null +++ b/packages/opencode/test/altimate/workspace/link-telemetry.test.ts @@ -0,0 +1,403 @@ +import { afterAll, afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs" +import os from "os" +import path from "path" + +// Set before the modules under test load: AltimateApi reads Global.Path.home at call time, +// but the sandbox must exist before the first credential read. +const ORIGINAL_TEST_HOME = process.env.OPENCODE_TEST_HOME +const SANDBOX = mkdtempSync(path.join(os.tmpdir(), "link-telemetry-")) +process.env.OPENCODE_TEST_HOME = path.join(SANDBOX, "home") +mkdirSync(path.join(SANDBOX, "home", ".altimate"), { recursive: true }) +const API_URL = "https://api.example.test" +writeFileSync( + path.join(SANDBOX, "home", ".altimate", "altimate.json"), + JSON.stringify({ altimateUrl: API_URL, altimateInstanceName: "acme", altimateApiKey: "test-key" }), +) + +const { + WorkspaceApi, + linkFailureReason, + endpointOf, + altimateRequest, + WorkspaceApiError, + NotFoundError, + ForbiddenError, + ConflictError, +} = await import("../../../src/altimate/workspace/api-client") +const { Telemetry } = await import("../../../src/altimate/telemetry") + +type Reply = { status: number; body?: unknown } | { throw: Error } | { hang: true } +type Event = import("../../../src/altimate/telemetry").Telemetry.Event + +describe("workspace link telemetry", () => { + const ORIGINAL_FETCH = globalThis.fetch + let events: Event[] + let requests: { method: string; url: string; body: unknown }[] + let replies: Reply[] + let trackSpy: ReturnType + + const binding = (id: number) => ({ + binding: { id: 1, datamate_id: id, datamate_name: `ws-${id}`, repo_remote: "git@x:y.git", project_path: null }, + }) + const of = (type: string) => events.filter((e) => e.type === type) as any[] + + beforeEach(() => { + events = [] + requests = [] + replies = [] + trackSpy = spyOn(Telemetry, "track").mockImplementation((e) => { + events.push(e) + }) + globalThis.fetch = (async (input: string | URL, init?: RequestInit) => { + requests.push({ + method: init?.method ?? "GET", + url: String(input), + body: init?.body ? JSON.parse(String(init.body)) : undefined, + }) + const r = replies.shift() ?? { status: 200, body: binding(1) } + if ("throw" in r) throw r.throw + if ("hang" in r) { + await new Promise((_, reject) => + init?.signal?.addEventListener("abort", () => + reject(Object.assign(new Error("aborted"), { name: "AbortError" })), + ), + ) + } + const rr = r as { status: number; body?: unknown } + return new Response(rr.body === undefined ? "" : JSON.stringify(rr.body), { + status: rr.status, + headers: { "content-type": "application/json" }, + }) + }) as typeof globalThis.fetch + }) + + afterEach(() => { + trackSpy.mockRestore() + globalThis.fetch = ORIGINAL_FETCH + }) + + afterAll(() => { + if (ORIGINAL_TEST_HOME === undefined) delete process.env.OPENCODE_TEST_HOME + else process.env.OPENCODE_TEST_HOME = ORIGINAL_TEST_HOME + rmSync(SANDBOX, { recursive: true, force: true }) + }) + + test("bind to an existing workspace reports workspace_linked on the success response", async () => { + replies = [{ status: 200, body: binding(42) }] + await WorkspaceApi.bindExisting(42, { repoRemote: "git@x:y.git" }) + expect(of("workspace_linked")).toEqual([expect.objectContaining({ workspace_id: 42, how: "bind_existing" })]) + expect(of("workspace_link_failed")).toHaveLength(0) + expect(of("workspace_api_error")).toHaveLength(0) + }) + + test("create-and-bind reports workspace_created then workspace_linked", async () => { + replies = [ + { status: 200, body: { ...binding(7), datamate: { id: 7, name: "ws-7" }, manage_url: "https://app/ws/7" } }, + ] + await WorkspaceApi.createAndBind({ name: "ws-7", identifier: { projectPath: "/p" } }) + expect(of("workspace_created")).toEqual([expect.objectContaining({ workspace_id: 7, how: "create_and_bind" })]) + expect(of("workspace_linked")).toEqual([expect.objectContaining({ workspace_id: 7, how: "create_and_bind" })]) + expect(events.findIndex((e) => e.type === "workspace_created")).toBeLessThan( + events.findIndex((e) => e.type === "workspace_linked"), + ) + }) + + test("an unbound create (the create-then-rebind path) reports workspace_created with how create_unbound", async () => { + replies = [{ status: 200, body: { id: 13 } }] + await WorkspaceApi.createWorkspaceUnbound({ name: "ws-13" }) + expect(of("workspace_created")).toEqual([expect.objectContaining({ workspace_id: 13, how: "create_unbound" })]) + expect(of("workspace_linked")).toHaveLength(0) // the rebind that follows reports the link + + events = [] + replies = [{ status: 200, body: { id: "7" } }] // malformed id: refused, and not reported as created + await expect(WorkspaceApi.createWorkspaceUnbound({ name: "x" })).rejects.toThrow(/no usable id/) + expect(of("workspace_created")).toHaveLength(0) + // ...but it is a failed link step: the answer was unusable, which is not a network failure. + expect(of("workspace_link_failed")).toEqual([expect.objectContaining({ how: "create_unbound", reason: "other" })]) + + events = [] + replies = [{ status: 409, body: { detail: { message: "name taken" } } }] + await expect(WorkspaceApi.createWorkspaceUnbound({ name: "x" })).rejects.toBeInstanceOf(ConflictError) + expect(of("workspace_link_failed")).toEqual([ + expect.objectContaining({ how: "create_unbound", reason: "conflict" }), + ]) + }) + + test("a rebind reports workspace_switched with both ids, then workspace_linked", async () => { + replies = [{ status: 200, body: binding(9) }] + await WorkspaceApi.rebindByRemote({ remote: "git@x:y.git", targetDatamateId: 9, expectedCurrentDatamateId: 4 }) + expect(of("workspace_switched")).toEqual([ + expect.objectContaining({ to_workspace_id: 9, from_workspace_id: 4, how: "rebind_remote" }), + ]) + expect(of("workspace_linked")).toEqual([expect.objectContaining({ workspace_id: 9, how: "rebind_remote" })]) + + events = [] + replies = [{ status: 200, body: binding(11) }] + await WorkspaceApi.rebindByPath({ projectPath: "/p", targetDatamateId: 11 }) + const sw = of("workspace_switched")[0] + expect(sw.to_workspace_id).toBe(11) + expect(sw.how).toBe("rebind_path") + expect("from_workspace_id" in sw).toBe(false) + }) + + test("unlink reports workspace_unlinked with the id the caller knew", async () => { + replies = [{ status: 204 }] + expect(await WorkspaceApi.unbindProject({ repoRemote: "git@x:y.git" }, 42)).toBe(true) + expect(of("workspace_unlinked")).toEqual([expect.objectContaining({ workspace_id: 42 })]) + + events = [] + replies = [{ status: 404, body: { detail: "no binding" } }] + expect(await WorkspaceApi.unbindProject({ repoRemote: "git@x:y.git" })).toBe(false) + expect(of("workspace_unlinked")).toHaveLength(0) + // "Already unlinked" is an answer, not a failed call. + expect(of("workspace_api_error")).toHaveLength(0) + + events = [] + replies = [{ status: 500, body: { detail: "boom" } }] + await expect(WorkspaceApi.unbindProject({ repoRemote: "git@x:y.git" })).rejects.toBeInstanceOf(WorkspaceApiError) + expect(of("workspace_api_error")).toEqual([ + expect.objectContaining({ endpoint: "DELETE /datamate-project-bindings/", status: 500, status_class: "5xx" }), + ]) + }) + + test("failed links report workspace_link_failed with a reason class and the API error", async () => { + replies = [{ status: 404, body: { detail: "gone" } }] + await expect(WorkspaceApi.bindExisting(42, { projectPath: "/p" })).rejects.toBeInstanceOf(NotFoundError) + expect(of("workspace_link_failed")).toEqual([ + expect.objectContaining({ how: "bind_existing", reason: "not_found", workspace_id: 42 }), + ]) + expect(of("workspace_api_error")).toEqual([ + expect.objectContaining({ endpoint: "POST /datamate-project-bindings/bind", status: 404, workspace_id: 42 }), + ]) + expect(of("workspace_linked")).toHaveLength(0) + + events = [] + replies = [{ status: 403, body: { detail: "nope" } }] + await expect(WorkspaceApi.bindExisting(42, { projectPath: "/p" })).rejects.toBeInstanceOf(ForbiddenError) + expect(of("workspace_link_failed")[0].reason).toBe("auth") + // A 403 is a refusal, not an expired session. + expect(of("workspace_auth_expired")).toHaveLength(0) + expect(of("workspace_api_error")).toEqual([ + expect.objectContaining({ status: 403, status_class: "4xx", workspace_id: 42 }), + ]) + + events = [] + replies = [{ status: 401, body: { detail: "expired" } }] + await expect(WorkspaceApi.bindExisting(42, { projectPath: "/p" })).rejects.toBeInstanceOf(WorkspaceApiError) + expect(of("workspace_link_failed")[0].reason).toBe("auth") + expect(of("workspace_auth_expired")).toEqual([ + expect.objectContaining({ endpoint: "POST /datamate-project-bindings/bind", status: 401, workspace_id: 42 }), + ]) + expect(of("workspace_api_error")).toHaveLength(0) + + events = [] + replies = [{ status: 409, body: { detail: { message: "already linked", existing_datamate_id: 3 } } }] + await expect(WorkspaceApi.bindExisting(42, { projectPath: "/p" })).rejects.toBeInstanceOf(ConflictError) + expect(of("workspace_link_failed")[0].reason).toBe("conflict") + + events = [] + replies = [{ throw: new Error("ECONNREFUSED") }] + await expect(WorkspaceApi.rebindByPath({ projectPath: "/p", targetDatamateId: 5 })).rejects.toBeInstanceOf( + WorkspaceApiError, + ) + expect(of("workspace_link_failed")[0]).toEqual(expect.objectContaining({ how: "rebind_path", reason: "network" })) + expect(of("workspace_api_error")[0]).toEqual( + expect.objectContaining({ + endpoint: "PUT /datamate-project-bindings/by-path", + status_class: "network", + workspace_id: 5, + }), + ) + + events = [] + replies = [{ status: 500, body: { detail: "boom" } }] + await expect(WorkspaceApi.createAndBind({ name: "x", identifier: { projectPath: "/p" } })).rejects.toBeInstanceOf( + WorkspaceApiError, + ) + expect(of("workspace_link_failed")[0]).toEqual(expect.objectContaining({ how: "create_and_bind", reason: "other" })) + expect(of("workspace_api_error")[0].status_class).toBe("5xx") + }) + + test("endpoints are masked of ids and a timeout is its own class", async () => { + replies = [{ status: 500, body: { detail: "x" } }] + await expect(WorkspaceApi.getBindingForRemote("git@x:y.git")).rejects.toBeInstanceOf(WorkspaceApiError) + expect(of("workspace_link_failed")).toHaveLength(0) + expect(of("workspace_api_error")[0].endpoint).toBe("GET /datamate-project-bindings/by-remote") + + events = [] + replies = [{ hang: true }] + const r = WorkspaceApi.listDatamates() + await expect(r).rejects.toThrow(/timed out/) + expect(of("workspace_api_error")[0]).toEqual(expect.objectContaining({ status_class: "timeout" })) + }, 40_000) + + test("endpoints mask every dynamic segment, and a bad 2xx body is its own class", async () => { + expect(endpointOf("GET", "/datamate-project-bindings", "/by-remote")).toBe( + "GET /datamate-project-bindings/by-remote", + ) + // The datamates router only lists its root here: any other shape is masked whole. + expect(endpointOf("GET", "/datamates", "/123/memory/8f2c0a1e-9b7d-4e11-a0c2-5d6e7f8a9b0c")).toBe( + "GET /datamates/:id/:id/:id", + ) + expect(endpointOf("POST", "/datamates", "/42/skills/pub_Ab12Cd/attach")).toBe( + "POST /datamates/:id/:id/:id/:id", + ) + // A route table, not spelling: ids and file paths are masked whatever they contain, including + // a public id or a file named after a route word. + expect(endpointOf("GET", "/skills", "/pub-many")).toBe("GET /skills/:id") + expect(endpointOf("GET", "/skills", "/status")).toBe("GET /skills/:id") + expect(endpointOf("PUT", "/skills", "/status/datamates")).toBe("PUT /skills/:id/datamates") + expect(endpointOf("GET", "/skills", "/pub_x/files/status")).toBe("GET /skills/:id/files/:path") + expect(endpointOf("GET", "/skills", "/pub_x/files/docs/readme.md")).toBe("GET /skills/:id/files/:path") + expect(endpointOf("GET", "/skills", "/pub_x/unexpected/route")).toBe("GET /skills/:id/:id/:id") + expect(endpointOf("PATCH", "/datamates/memory", "/list")).toBe("PATCH /datamates/memory/list") + expect(endpointOf("PATCH", "/datamates/memory", "/mem_status")).toBe("PATCH /datamates/memory/:id") + expect(endpointOf("POST", "/datamate-project-bindings", "/bind")).toBe("POST /datamate-project-bindings/bind") + expect(endpointOf("DELETE", "/datamate-project-bindings", "/")).toBe("DELETE /datamate-project-bindings/") + // Not a workspace router (whoami reports nothing), so not listed: masked whole. + expect(endpointOf("GET", "/users", "/me")).toBe("GET /:id/:id") + expect(endpointOf("GET", "/unknown-router", "/anything/else")).toBe("GET /:id/:id/:id") + expect(endpointOf("GET", "/datamates", "/42/skills/pub_Ab12Cd/files/readme.md")).toBe( + "GET /datamates/:id/:id/:id/:id/:id", + ) + + replies = [{ status: 200 }] // 2xx with an empty body where JSON was required + await expect(WorkspaceApi.listDatamates()).rejects.toThrow(/Empty 200 body/) + expect(of("workspace_api_error")).toEqual([expect.objectContaining({ status_class: "response", status: 200 })]) + }) + + test("a body that stalls after the server answered keeps the answer's status class", async () => { + globalThis.fetch = (async (_input: string | URL | Request, init?: RequestInit) => + new Response( + new ReadableStream({ + start(controller) { + init?.signal?.addEventListener("abort", () => + controller.error(Object.assign(new Error("aborted"), { name: "AbortError" })), + ) + }, + }), + { status: 500 }, + )) as typeof globalThis.fetch + const stalled: any = await altimateRequest("GET", "/x", { base: "/datamates", timeoutMs: 200 }).catch((e) => e) + expect(stalled).toBeInstanceOf(WorkspaceApiError) + expect(stalled.message).toMatch(/timed out/) + expect(stalled.status).toBe(500) // the answer's status rides on the error + expect(linkFailureReason(stalled)).toBe("other") // answered: not a network failure + // A known status whose body failed classifies like a readable answer with that status. + expect(linkFailureReason(new WorkspaceApiError("stalled", 404))).toBe("not_found") + expect(linkFailureReason(new WorkspaceApiError("stalled", 403))).toBe("auth") + expect(linkFailureReason(new WorkspaceApiError("stalled", 409))).toBe("conflict") + expect(linkFailureReason(new WorkspaceApiError("stalled", 412))).toBe("conflict") + expect(linkFailureReason(new WorkspaceApiError("cannot reach"))).toBe("network") + expect(linkFailureReason(new WorkspaceApiError("no usable id", undefined, "response"))).toBe("other") + expect(of("workspace_api_error")).toEqual([expect.objectContaining({ status_class: "5xx", status: 500 })]) + + events = [] + globalThis.fetch = (async (_input: string | URL | Request, init?: RequestInit) => + new Response( + new ReadableStream({ + start(controller) { + init?.signal?.addEventListener("abort", () => + controller.error(Object.assign(new Error("aborted"), { name: "AbortError" })), + ) + }, + }), + { status: 200 }, + )) as typeof globalThis.fetch + await expect(altimateRequest("GET", "/x", { base: "/datamates", timeoutMs: 200 })).rejects.toThrow(/timed out/) + expect(of("workspace_api_error")).toEqual([expect.objectContaining({ status_class: "timeout", status: 200 })]) + }) + + test("a body that fails after the server answered keeps the answer's status class", async () => { + // Oversized 500: readBounded() refuses the body before the status branch; the row is still a 5xx. + globalThis.fetch = (async (_input: string | URL | Request, _init?: RequestInit) => + new Response("x", { status: 500, headers: { "content-length": "100000000" } })) as typeof globalThis.fetch + await expect(altimateRequest("GET", "/x", { base: "/datamates", boundResponse: true })).rejects.toBeInstanceOf( + WorkspaceApiError, + ) + expect(of("workspace_api_error")).toEqual([expect.objectContaining({ status_class: "5xx", status: 500 })]) + + events = [] + globalThis.fetch = (async (_input: string | URL | Request, _init?: RequestInit) => + new Response("x", { status: 200, headers: { "content-length": "100000000" } })) as typeof globalThis.fetch + await expect(altimateRequest("GET", "/x", { base: "/datamates", boundResponse: true })).rejects.toBeInstanceOf( + WorkspaceApiError, + ) + expect(of("workspace_api_error")).toEqual([expect.objectContaining({ status_class: "response", status: 200 })]) + }) + + test("a not-linked lookup is an answer: a 404 from either lookup reports no row", async () => { + replies = [ + { status: 404, body: { detail: "no binding" } }, + { status: 404, body: { detail: "no binding" } }, + ] + await expect( + WorkspaceApi.getBindingForProject({ repoRemote: "git@x:y.git", projectPath: "/p" }), + ).resolves.toBeNull() + expect(requests.map((r) => new URL(r.url).pathname)).toEqual([ + "/datamate-project-bindings/by-remote", + "/datamate-project-bindings/by-path", + ]) + expect(events).toEqual([]) + }) + + test("a declared status reports nothing; any other failure of the same call still does", async () => { + replies = [{ status: 404, body: { detail: "gone" } }] + await expect( + altimateRequest("GET", "/x", { base: "/skills", expectedStatuses: [404] }), + ).rejects.toBeInstanceOf(NotFoundError) + expect(events).toEqual([]) + + replies = [{ status: 409, body: { detail: "taken" } }] + await expect( + altimateRequest("GET", "/x", { base: "/skills", expectedStatuses: [404] }), + ).rejects.toBeInstanceOf(ConflictError) + expect(of("workspace_api_error")).toEqual([expect.objectContaining({ status: 409, status_class: "4xx" })]) + }) + + test("whoami is not a workspace call: its failures report no row", async () => { + for (const status of [401, 403, 404, 500]) { + replies = [{ status, body: { detail: "x" } }] + await expect(WorkspaceApi.whoami()).rejects.toBeInstanceOf(Error) + } + replies = [{ throw: new Error("ECONNREFUSED") }] + await expect(WorkspaceApi.whoami()).rejects.toBeInstanceOf(WorkspaceApiError) + expect(events).toEqual([]) + }) + + test("the 409 -> rebind fallback reports the failed bind, then the switch (rows as they are today)", async () => { + // `link` with a failed pre-check: bindExisting answers 409, and the command retries as a rebind on + // the identifier the conflict names. Pins the rows this sequence emits; the recovered conflict + // still counts one link_failed (disclosed, follow-up). + replies = [ + { status: 409, body: { detail: { message: "already linked", project_path: "/p" } } }, + { status: 200, body: binding(42) }, + ] + await expect(WorkspaceApi.bindExisting(42, { projectPath: "/p" })).rejects.toBeInstanceOf(ConflictError) + await WorkspaceApi.rebindByPath({ projectPath: "/p", targetDatamateId: 42 }) + expect(events.map((e) => e.type)).toEqual([ + "workspace_api_error", + "workspace_link_failed", + "workspace_switched", + "workspace_linked", + ]) + expect(of("workspace_api_error")[0]).toEqual(expect.objectContaining({ status: 409, workspace_id: 42 })) + expect(of("workspace_link_failed")[0]).toEqual( + expect.objectContaining({ how: "bind_existing", reason: "conflict", workspace_id: 42 }), + ) + expect(of("workspace_switched")[0]).toEqual(expect.objectContaining({ to_workspace_id: 42, how: "rebind_path" })) + expect("from_workspace_id" in of("workspace_switched")[0]).toBe(false) + expect(of("workspace_linked")[0]).toEqual(expect.objectContaining({ workspace_id: 42, how: "rebind_path" })) + }) + + test("linkFailureReason classifies every error the client throws", () => { + expect(linkFailureReason(new ForbiddenError("f"))).toBe("auth") + expect(linkFailureReason(new WorkspaceApiError("unauthorized", 401))).toBe("auth") + expect(linkFailureReason(new NotFoundError("n"))).toBe("not_found") + expect(linkFailureReason(new ConflictError({ message: "c" }))).toBe("conflict") + expect(linkFailureReason(new WorkspaceApiError("Cannot reach x: ECONNRESET"))).toBe("network") + expect(linkFailureReason(new WorkspaceApiError("bad", 500))).toBe("other") + expect(linkFailureReason(new Error("whatever"))).toBe("other") + }) +}) diff --git a/packages/opencode/test/altimate/workspace/memory-sync.test.ts b/packages/opencode/test/altimate/workspace/memory-sync.test.ts index c2a0e3c8d2..b1d20fa7c5 100644 --- a/packages/opencode/test/altimate/workspace/memory-sync.test.ts +++ b/packages/opencode/test/altimate/workspace/memory-sync.test.ts @@ -6,7 +6,7 @@ // the mirror actually issues — method, path, body — rather than a mock's call // log. Cases claiming "nothing was sent" check a zero request count, not merely // the absence of a throw. -import { afterAll, afterEach, beforeEach, describe, expect, test } from "bun:test" +import { afterAll, afterEach, beforeEach, describe, expect, test, spyOn } from "bun:test" import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs" import path from "node:path" import os from "node:os" @@ -54,9 +54,7 @@ const { toBlock, whenHydrated, } = await import("../../../src/altimate/workspace/memory-sync") -const { MIRROR_SOURCE, extractRecordId, LIST_LIMIT } = await import( - "../../../src/altimate/workspace/memory-api" -) +const { MIRROR_SOURCE, extractRecordId, LIST_LIMIT } = await import("../../../src/altimate/workspace/memory-api") import { AltimateApi } from "../../../src/altimate/api/client" @@ -124,7 +122,8 @@ function stubFetch() { if (method === "POST") { const body = init?.body ? JSON.parse(init.body) : {} for (const rec of Array.isArray(createResult) ? createResult : []) { - if (rec?.id) listResponse.push({ id: rec.id, memory: body.messages?.[0]?.content ?? "", metadata: body.metadata }) + if (rec?.id) + listResponse.push({ id: rec.id, memory: body.messages?.[0]?.content ?? "", metadata: body.metadata }) } } return { message: "ok", result: createResult } @@ -132,8 +131,7 @@ function stubFetch() { if (url.includes("/datamates/")) return { datamates: workspaces } return { message: "ok" } })() - const status = - url.includes("/datamate-project-bindings/by-") && !serverBinding ? 404 : 200 + const status = url.includes("/datamate-project-bindings/by-") && !serverBinding ? 404 : 200 return new Response(JSON.stringify(payload), { status, headers: { "Content-Type": "application/json" }, @@ -198,10 +196,8 @@ afterEach(() => { // seam fall through to the real lookup, so a leaked value from a previous // test would decide their outcome. serverBinding = null - ;(AltimateApi as unknown as { isConfigured: typeof originalIsConfigured }).isConfigured = - originalIsConfigured - ;(AltimateApi as unknown as { getCredentials: typeof originalGetCreds }).getCredentials = - originalGetCreds + ;(AltimateApi as unknown as { isConfigured: typeof originalIsConfigured }).isConfigured = originalIsConfigured + ;(AltimateApi as unknown as { getCredentials: typeof originalGetCreds }).getCredentials = originalGetCreds delete process.env.ALTIMATE_DISABLE_WORKSPACE delete syncInternals.resolveBinding delete syncInternals.blockExists @@ -595,7 +591,11 @@ describe("mirrorBlock", () => { createResult = [{ id: "mem-archive-late" }] await mirrorBlock(b) listResponse = [ - { id: "mem-archive-late", memory: b.content, metadata: { source: MIRROR_SOURCE, block_id: "to-archive-late", block_scope: "global" } }, + { + id: "mem-archive-late", + memory: b.content, + metadata: { source: MIRROR_SOURCE, block_id: "to-archive-late", block_scope: "global" }, + }, ] const net = gatedFetch() let settled = false @@ -907,8 +907,7 @@ describe("mirrorBlock", () => { // incrementApplied rewrites a training block on EVERY session start to bump // a counter inside the content body. Mirroring that would cost a request // per training block per session. - const meta = (applied: number) => - `\nAlways alias CTEs.` + const meta = (applied: number) => `\nAlways alias CTEs.` const b = block({ id: "training/rule/aliases", tags: ["training", "rule"], content: meta(1) }) await mirrorBlock(b) captured = [] @@ -955,7 +954,7 @@ describe("memory_enabled", () => { expect(callsTo("/datamates/memory/", "POST").length).toBe(0) workspaces = [{ id: 42, name: "acme", memory_enabled: true }] - serverBinding = null + serverBinding = null captured = [] await mirrorBlock(block({ id: "after-enable" })) expect(callsTo("/datamates/memory/", "POST").length).toBe(1) @@ -1117,10 +1116,7 @@ describe("tag encoding", () => { // The previous comma-joined form split this tag into two on read. const tags = ["warehouse", "owner: data, platform", "prod"] const meta = buildMetadata(block({ tags }), null) - const back = toBlock( - { id: "1", memory: "x", metadata: { ...meta, source: MIRROR_SOURCE } } as any, - undefined, - ) + const back = toBlock({ id: "1", memory: "x", metadata: { ...meta, source: MIRROR_SOURCE } } as any, undefined) expect(back?.tags).toEqual(tags) }) @@ -1180,9 +1176,7 @@ describe("reaping blocks archived elsewhere", () => { const seeThrough = (blocks: Record) => { const removed: { id: string; scope: string; directory?: string; expectUpdated?: string }[] = [] syncInternals.readBlock = async (scope, id) => - blocks[id] - ? ({ id, scope, content: "c", tags: [], created: "", updated: blocks[id].updated } as any) - : undefined + blocks[id] ? ({ id, scope, content: "c", tags: [], created: "", updated: blocks[id].updated } as any) : undefined syncInternals.removeBlock = async (scope, id, directory, expectUpdated) => { removed.push({ id, scope, directory, expectUpdated }) return true @@ -1310,7 +1304,11 @@ describe("reaping blocks archived elsewhere", () => { const removed = seeThrough({ split: { updated: "2026-05-01T00:00:00.000Z" } }) listResponse = [ // The live primary. - { id: "mem-primary", memory: "kept", metadata: { source: MIRROR_SOURCE, block_id: "split", block_scope: "global" } }, + { + id: "mem-primary", + memory: "kept", + metadata: { source: MIRROR_SOURCE, block_id: "split", block_scope: "global" }, + }, // The extra, archived at push time. archived("split", { archived_at: "2026-06-01T00:00:00.000Z" }), ] @@ -1509,7 +1507,11 @@ describe("hydrate", () => { test("drops archived records", async () => { listResponse = [ { id: "1", memory: "live", metadata: { source: MIRROR_SOURCE, block_id: "live", block_scope: "global" } }, - { id: "2", memory: "gone", metadata: { source: MIRROR_SOURCE, block_id: "gone", block_scope: "global", archived: "true" } }, + { + id: "2", + memory: "gone", + metadata: { source: MIRROR_SOURCE, block_id: "gone", block_scope: "global", archived: "true" }, + }, ] await hydrate(SES) expect(overlayBlocks(SES).map((b) => b.id)).toEqual(["live"]) @@ -1518,7 +1520,11 @@ describe("hydrate", () => { test("drops project records belonging to another workspace", async () => { listResponse = [ { id: "1", memory: "g", metadata: { source: MIRROR_SOURCE, block_id: "g", block_scope: "global" } }, - { id: "2", memory: "other", metadata: { source: MIRROR_SOURCE, block_id: "other", block_scope: "project", datamate_id: "999" } }, + { + id: "2", + memory: "other", + metadata: { source: MIRROR_SOURCE, block_id: "other", block_scope: "project", datamate_id: "999" }, + }, ] await hydrate(SES) expect(overlayBlocks(SES).map((b) => b.id)).toEqual(["g"]) @@ -1592,7 +1598,11 @@ describe("archiveBlock", () => { await mirrorBlock(b) captured = [] listResponse = [ - { id: "mem-archive", memory: b.content, metadata: { source: MIRROR_SOURCE, block_id: "to-archive", block_scope: "global" } }, + { + id: "mem-archive", + memory: b.content, + metadata: { source: MIRROR_SOURCE, block_id: "to-archive", block_scope: "global" }, + }, ] await archiveBlock("global", "to-archive") @@ -1730,6 +1740,28 @@ describe("backfill", () => { block({ id: `${prefix}/${i}`, content: `Block ${prefix} ${i} describes a warehouse convention.` }), ) + test("every block stored in the workspace reports workspace_memory_contributed with the workspace id", async () => { + const { Telemetry } = await import("../../../src/altimate/telemetry") + const events: any[] = [] + const spy = spyOn(Telemetry, "track").mockImplementation((e) => { + events.push(e) + }) + try { + createResult = [{ id: "mem-contrib" }] + // Project-scope blocks are the ones bound to the workspace; global memory is the account's. + const bound = blocks("contrib", 2).map((b) => ({ ...b, scope: "project" })) + const result = await backfill(bound, BINDING as any) + expect(result.ok).toBe(2) + const contributed = events.filter((e) => e.type === "workspace_memory_contributed") + expect(contributed).toEqual([ + expect.objectContaining({ workspace_id: 42, block_count: 1 }), + expect.objectContaining({ workspace_id: 42, block_count: 1 }), + ]) + } finally { + spy.mockRestore() + } + }) + test("reads the record set once for the whole sweep, not once per block", async () => { // Every block in a first bind is an index miss, so resolving each through // its own lookup made a bind cost one full fetch per block. @@ -1906,7 +1938,11 @@ describe("binding changes", () => { // The first lookup adopts (and notifies); later ones read the cache, as in production. if (!adopted) { adopted = true - await recordApprovedBinding(dir, { ...BINDING, projectPath: dir, linkedAt: 3 }, { awaitBackfill: true, seed: false }) + await recordApprovedBinding( + dir, + { ...BINDING, projectPath: dir, linkedAt: 3 }, + { awaitBackfill: true, seed: false }, + ) } return BINDING as any } @@ -1937,7 +1973,11 @@ describe("refresh racing a relink", () => { await reachedList const { recordApprovedBinding } = await import("../../../src/altimate/workspace/state") const dir = mkdtempSync(path.join(SANDBOX, "race-")) - await recordApprovedBinding(dir, { ...BINDING, datamateId: 44, projectPath: dir, linkedAt: 4 }, { awaitBackfill: true, seed: false }) + await recordApprovedBinding( + dir, + { ...BINDING, datamateId: 44, projectPath: dir, linkedAt: 4 }, + { awaitBackfill: true, seed: false }, + ) release?.() const result = await pending // Superseded: neither its read nor the prior overlay belongs to the new binding. @@ -1961,7 +2001,11 @@ describe("overlay invalidation", () => { await hydrate(SES) expect(overlayBlocks(SES).length).toBe(1) const dir = mkdtempSync(path.join(SANDBOX, "hide-")) - await recordApprovedBinding(dir, { ...BINDING, datamateId: 45, projectPath: dir, linkedAt: 5 }, { awaitBackfill: true, seed: false }) + await recordApprovedBinding( + dir, + { ...BINDING, datamateId: 45, projectPath: dir, linkedAt: 5 }, + { awaitBackfill: true, seed: false }, + ) expect(overlayBlocks(SES)).toEqual([]) }) @@ -2004,7 +2048,11 @@ describe("epoch bracketing and scope", () => { calls++ // First lookup returns A, but the relink to B lands before it returns. if (calls === 1) { - await recordApprovedBinding(dir, { ...BINDING, datamateId: 46, projectPath: dir, linkedAt: 6 }, { awaitBackfill: true, seed: false }) + await recordApprovedBinding( + dir, + { ...BINDING, datamateId: 46, projectPath: dir, linkedAt: 6 }, + { awaitBackfill: true, seed: false }, + ) return BINDING as any } return { ...BINDING, datamateId: 46 } as any @@ -2026,10 +2074,18 @@ describe("epoch bracketing and scope", () => { const other = mkdtempSync(path.join(SANDBOX, "other-")) await refresh(SES, mine) expect(overlayBlocks(SES).map((x) => x.id)).toEqual(["mine"]) - await recordApprovedBinding(other, { ...BINDING, datamateId: 47, projectPath: other, linkedAt: 7 }, { awaitBackfill: true, seed: false }) + await recordApprovedBinding( + other, + { ...BINDING, datamateId: 47, projectPath: other, linkedAt: 7 }, + { awaitBackfill: true, seed: false }, + ) expect(overlayBlocks(SES).map((x) => x.id)).toEqual(["mine"]) // A change to this project's own binding still hides it. - await recordApprovedBinding(mine, { ...BINDING, datamateId: 48, projectPath: mine, linkedAt: 8 }, { awaitBackfill: true, seed: false }) + await recordApprovedBinding( + mine, + { ...BINDING, datamateId: 48, projectPath: mine, linkedAt: 8 }, + { awaitBackfill: true, seed: false }, + ) expect(overlayBlocks(SES)).toEqual([]) }) }) @@ -2088,9 +2144,7 @@ describe("session isolation and turn behaviour", () => { // The caller's enclosing block runs on EVERY user turn, not once per // session. Without idempotence a 20-turn conversation costs 40 requests, // and the first injection of every turn blocks on them. - listResponse = [ - { id: "1", memory: "x", metadata: { source: MIRROR_SOURCE, block_id: "x", block_scope: "global" } }, - ] + listResponse = [{ id: "1", memory: "x", metadata: { source: MIRROR_SOURCE, block_id: "x", block_scope: "global" } }] await hydrate(SES) await hydrate(SES) await hydrate(SES) @@ -2128,9 +2182,7 @@ describe("session isolation and turn behaviour", () => { }) test("overlayBlocks returns a copy, so a caller cannot corrupt the cache", async () => { - listResponse = [ - { id: "1", memory: "x", metadata: { source: MIRROR_SOURCE, block_id: "x", block_scope: "global" } }, - ] + listResponse = [{ id: "1", memory: "x", metadata: { source: MIRROR_SOURCE, block_id: "x", block_scope: "global" } }] await hydrate(SES) overlayBlocks(SES).length = 0 expect(overlayBlocks(SES).length).toBe(1) @@ -2189,9 +2241,7 @@ describe("binding resolution for the mirror", () => { await mirrorBlock(block(), dir) - const posts = captured.filter( - (c) => c.method === "POST" && c.url.includes("/datamates/memory/"), - ) + const posts = captured.filter((c) => c.method === "POST" && c.url.includes("/datamates/memory/")) expect(posts.length).toBe(1) }) @@ -2205,9 +2255,7 @@ describe("binding resolution for the mirror", () => { await mirrorBlock(block(), dir) - const posts = captured.filter( - (c) => c.method === "POST" && c.url.includes("/datamates/memory/"), - ) + const posts = captured.filter((c) => c.method === "POST" && c.url.includes("/datamates/memory/")) expect(posts.length).toBe(0) }) }) @@ -2627,7 +2675,11 @@ describe("sync state", () => { listResponse = [note("a", "Naming conventions", "2026-02-01T00:00:00.000Z"), note("c", "Cost alerts")] await refresh(`${SES}-state-3`, dir) const entity = await memoryState(dir, (e) => e.changes !== null) - expect(entity?.changes).toEqual({ added: ["Cost alerts"], removed: ["Warehouse sizing"], updated: ["Naming conventions"] }) + expect(entity?.changes).toEqual({ + added: ["Cost alerts"], + removed: ["Warehouse sizing"], + updated: ["Naming conventions"], + }) expect(entity?.count).toBe(2) }) @@ -2750,7 +2802,9 @@ describe("sync state", () => { listFails = false } const failed = await memoryState(dir, (e) => e.status === "error") - expect(failed).toMatchObject({ status: "error", count: 1, error: "could not load workspace memory" }) + expect(failed).toMatchObject({ status: "error", count: 1 }) + // The record keeps what failed, so the sync-failure reason class is real rather than "other". + expect(failed?.error).toMatch(/^could not load workspace memory: /) // "memory loaded Xm ago" reads the last success, which the failure did not move. expect(failed?.lastSucceededAt).toBe(loaded!.lastSucceededAt) }) @@ -2786,12 +2840,18 @@ describe("sync state", () => { // project is relinked. let release!: () => void const gate = new Promise((r) => (release = r)) - const holder = SyncState.record(dir, BINDING.datamateId, "memory", {}, { - isCurrent: async () => { - await gate - return false + const holder = SyncState.record( + dir, + BINDING.datamateId, + "memory", + {}, + { + isCurrent: async () => { + await gate + return false + }, }, - }) + ) listResponse = [] await refresh(`${SES}-state-late-2`, dir) // Unlink: every binding epoch moves, and the queued "nothing left" record is now stale. diff --git a/packages/opencode/test/altimate/workspace/precedence.test.ts b/packages/opencode/test/altimate/workspace/precedence.test.ts index 680b916965..93060cc717 100644 --- a/packages/opencode/test/altimate/workspace/precedence.test.ts +++ b/packages/opencode/test/altimate/workspace/precedence.test.ts @@ -4,7 +4,7 @@ // bound workspace's engine has attached. The binding and the engine-attribution read // both go through `precedenceInternals`, so these exercise the decision logic without // booting an instance, reading config, or touching MCP state. -import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" import { INTEGRATION_TYPE, MAX_TRACKED_SESSIONS, @@ -26,6 +26,7 @@ import { servedInventory, } from "../../../src/altimate/workspace/precedence" import * as Registry from "../../../src/altimate/native/connections/registry" +import { Telemetry } from "../../../src/altimate/telemetry" // altimate_change - shared with awareness.test.ts; see precedence-fixture.ts import { BIGQUERY_TOOLS, SNOWFLAKE_TOOLS, WAREHOUSE_CONFIGS, bindTo } from "./precedence-fixture" import { canonicalType } from "../../../src/altimate/native/connections/registry" @@ -1403,3 +1404,32 @@ describe("servedInventory — what the model will be told is routed", () => { }) }) // altimate_change end + +describe("telemetry — the precedence snapshot", () => { + test("a snapshot is reported when it changes for the session, not on every refresh", async () => { + const rows: Array> = [] + const spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e as Record) + }) + try { + await refresh(SESSION, SNOWFLAKE_TOOLS) + await refresh(SESSION, SNOWFLAKE_TOOLS) + const snapshots = () => rows.filter((r) => r.type === "workspace_engine_precedence") + expect(snapshots()).toEqual([ + expect.objectContaining({ enabled: true, shadowed_types: 1, escape_hatch: false, workspace_id: 42 }), + ]) + // One local snowflake connection in the fixture registry is shadowed; the four served + // capabilities are not what is counted. + expect(snapshots()[0].shadowed_connections).toBe(1) + await refresh(SESSION, {}) + expect(snapshots().at(-1)).toEqual( + expect.objectContaining({ enabled: false, reason: "nothing-materialised", shadowed_types: 0 }), + ) + await refresh("ses_other", SNOWFLAKE_TOOLS) + expect(snapshots()).toHaveLength(3) + expect(snapshots()[2].session_id).toBe("ses_other") + } finally { + spy.mockRestore() + } + }) +}) diff --git a/packages/opencode/test/altimate/workspace/starter.test.ts b/packages/opencode/test/altimate/workspace/starter.test.ts index 52a2104d1b..547393bea3 100644 --- a/packages/opencode/test/altimate/workspace/starter.test.ts +++ b/packages/opencode/test/altimate/workspace/starter.test.ts @@ -1,6 +1,7 @@ // altimate_change - new file -import { describe, expect, test } from "bun:test" -import { renderStarter, wrapStarter } from "../../../src/altimate/workspace/starter" +import { describe, expect, spyOn, test } from "bun:test" +import { renderStarter, trackStarterShown, wrapStarter } from "../../../src/altimate/workspace/starter" +import { Telemetry } from "../../../src/altimate/telemetry" import type { WorkspaceContents } from "../../../src/altimate/workspace/contents" const full: WorkspaceContents = { @@ -46,7 +47,12 @@ describe("renderStarter", () => { }) test("says what an empty or unsynced workspace means, and suggests only what it can back", () => { - const empty = renderStarter("Acme", { skills: [], integrations: [], memoryEnabled: false, knowledge: { kind: "off" } }) + const empty = renderStarter("Acme", { + skills: [], + integrations: [], + memoryEnabled: false, + knowledge: { kind: "off" }, + }) expect(empty.lines).toEqual([ "Skills: none yet. Add them in the workspace and they sync here.", "Knowledge: off for this workspace.", @@ -110,3 +116,31 @@ describe("wrapStarter", () => { expect(wrapStarter("a\n\nb c", 80)).toBe("a\n\nb c") }) }) + +describe("starter_shown telemetry", () => { + test("reports what the workspace provides as counts, and leaves unknowns out", () => { + const events: any[] = [] + const spy = spyOn(Telemetry, "track").mockImplementation((e) => { + events.push(e) + }) + try { + trackStarterShown(5, full) + expect(events).toEqual([ + expect.objectContaining({ + type: "starter_shown", + workspace_id: 5, + skills: Array.isArray(full.skills) ? full.skills.length : undefined, + integrations: Array.isArray(full.integrations) ? full.integrations.length : undefined, + knowledge: full.knowledge?.kind ?? "unknown", + memory_enabled: full.memoryEnabled === true, + }), + ]) + events.length = 0 + trackStarterShown(6, { skills: "unknown", integrations: null, memoryEnabled: null, knowledge: null }) + expect(events[0]).toEqual(expect.objectContaining({ workspace_id: 6, knowledge: "unknown" })) + for (const key of ["skills", "integrations", "memory_enabled"]) expect(key in events[0]).toBe(false) + } finally { + spy.mockRestore() + } + }) +}) diff --git a/packages/opencode/test/altimate/workspace/sync-state.test.ts b/packages/opencode/test/altimate/workspace/sync-state.test.ts index c4e275e6d9..1dfe37abb1 100644 --- a/packages/opencode/test/altimate/workspace/sync-state.test.ts +++ b/packages/opencode/test/altimate/workspace/sync-state.test.ts @@ -61,7 +61,11 @@ describe("diff", () => { }) test("identical sets have no changes", () => { - expect(SyncState.diff({ a: skill("Alpha") }, { a: skill("Alpha") })).toEqual({ added: [], removed: [], updated: [] }) + expect(SyncState.diff({ a: skill("Alpha") }, { a: skill("Alpha") })).toEqual({ + added: [], + removed: [], + updated: [], + }) }) test("an id named like an inherited property is still an item", () => { @@ -75,7 +79,13 @@ describe("record", () => { await SyncState.record(project, 7, "skills", { items: { a: skill("Alpha"), b: skill("Beta") } }) const state = await SyncState.read(project, 7) - expect(state?.entities.skills).toMatchObject({ kind: "skills", status: "ok", count: 2, lastChangedAt: null, changes: null }) + expect(state?.entities.skills).toMatchObject({ + kind: "skills", + status: "ok", + count: 2, + lastChangedAt: null, + changes: null, + }) expect(state?.entities.skills?.lastCheckedAt).toBeNumber() expect(state?.entities.skills?.lastSucceededAt).toBe(state!.entities.skills!.lastCheckedAt) expect(published).toEqual([]) @@ -196,7 +206,13 @@ describe("record", () => { test("a late result from a previous binding is dropped", async () => { await SyncState.record(project, 8, "memory", { items: { m1: skill("New workspace block") } }) - await SyncState.record(project, 7, "memory", { error: "could not load workspace memory" }, { isCurrent: () => false }) + await SyncState.record( + project, + 7, + "memory", + { error: "could not load workspace memory" }, + { isCurrent: () => false }, + ) const state = await SyncState.read(project, 8) expect(state?.entities.memory).toMatchObject({ status: "ok", count: 1 }) @@ -298,3 +314,37 @@ describe("forget", () => { expect((await SyncState.read(project, 7))?.entities.memory?.count).toBe(1) }) }) + +describe("workspace_sync_failed telemetry", () => { + test("a kind flipping to error reports once, with a reason class; a repeat does not", async () => { + const { Telemetry } = await import("../../../src/altimate/telemetry") + const events: any[] = [] + const spy = spyOn(Telemetry, "track").mockImplementation((e) => { + events.push(e) + }) + try { + await SyncState.record(project, 9, "memory", { error: "401 unauthorized" }) + expect(events).toEqual([ + expect.objectContaining({ type: "workspace_sync_failed", workspace_id: 9, kind: "memory", reason: "auth" }), + ]) + await SyncState.record(project, 9, "memory", { error: "still unauthorized" }) // already in error: no new row + expect(events).toHaveLength(1) + await SyncState.record(project, 9, "memory", { items: { a: skill("Alpha") } }) // back to ok + await SyncState.record(project, 9, "memory", { error: "Cannot reach https://api: ECONNREFUSED" }) + expect(events).toHaveLength(2) + expect(events[1]).toEqual(expect.objectContaining({ kind: "memory", reason: "network" })) + } finally { + spy.mockRestore() + } + }) + + test("syncFailureReason classifies the messages the sync produces", () => { + expect(SyncState.syncFailureReason("Request failed with status 403")).toBe("auth") + expect(SyncState.syncFailureReason("Please sign in again")).toBe("auth") + expect(SyncState.syncFailureReason("Workspace not found (404)")).toBe("not_found") + expect(SyncState.syncFailureReason("Request to x timed out after 15s")).toBe("network") + expect(SyncState.syncFailureReason("TypeError: fetch failed")).toBe("network") + expect(SyncState.syncFailureReason("The server returned no usable id")).toBe("other") + expect(SyncState.syncFailureReason(undefined)).toBe("other") + }) +}) diff --git a/packages/opencode/test/altimate/workspace/tool-call-telemetry.test.ts b/packages/opencode/test/altimate/workspace/tool-call-telemetry.test.ts new file mode 100644 index 0000000000..f4d38eb31c --- /dev/null +++ b/packages/opencode/test/altimate/workspace/tool-call-telemetry.test.ts @@ -0,0 +1,121 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" +import os from "os" +import { Telemetry } from "../../../src/altimate/telemetry" +import { isWorkspaceClient, trackWorkspaceToolCall } from "../../../src/altimate/workspace/tool-call-telemetry" + +describe("workspace integration tool calls", () => { + let rows: any[] + let spy: ReturnType + beforeEach(() => { + rows = [] + spy = spyOn(Telemetry, "track").mockImplementation((e) => { + rows.push(e) + }) + Telemetry.endSession("s1") + }) + afterEach(() => { + spy.mockRestore() + Telemetry.endSession("s1") + Telemetry.endSession("s0") + // The singleton context is process-wide: left at s1/42 it would stamp a later suite's + // session-less rows. + Telemetry.setContext({ sessionId: "", projectId: "" }) + }) + + const call = ( + clientName: string, + outcome: "success" | "error", + error?: unknown, + workspaceId: number | undefined = 42, + ) => + trackWorkspaceToolCall({ + key: `${clientName}_jira_get_issue`, + clientName, + sessionID: "s1", + messageID: "m1", + workspaceId, + startedAt: Date.now() - 5, + outcome, + error, + }) + + test("the engine client's calls become tool_call rows with via_workspace; a success counts as an integration use", () => { + expect(call("datamate", "success")).toBe(true) + expect(rows).toHaveLength(1) + expect(rows[0]).toEqual( + expect.objectContaining({ + type: "tool_call", + tool_name: "datamate_jira_get_issue", + tool_type: "mcp", + status: "success", + via_workspace: true, + workspace_id: 42, + session_id: "s1", + message_id: "m1", + }), + ) + expect(rows[0].duration_ms).toBeGreaterThanOrEqual(0) + expect("error" in rows[0]).toBe(false) + expect(Telemetry.workspaceUseOf("s1").integrations).toBe(1) + }) + + test("a failed call is a tool_call error row with a masked message and no integration use", () => { + expect(call("datamate", "error", new Error("boom at /home/someone/secret.txt"))).toBe(true) + expect(rows[0]).toEqual(expect.objectContaining({ status: "error", via_workspace: true, tool_type: "mcp" })) + expect(rows[0].error).not.toContain("/home/someone") + expect(Telemetry.workspaceUseOf("s1").integrations).toBe(0) + expect(call("datamate", "error", "string failure with token=abc123secret and https://u:pw@host/x?q=1")).toBe(true) + expect(rows[1].error).toContain("string failure") + expect(rows[1].error).not.toContain("abc123secret") + expect(rows[1].error).not.toContain("u:pw@") + }) + + test("a failed call's message loses the user name and host name too", () => { + const user = os.userInfo().username + const host = os.hostname() + expect(call("datamate", "error", `login failed for user=${user} on ${host}`)).toBe(true) + if (user.length > 2) expect(rows[0].error).not.toContain(`=${user}`) + if (host.length > 2) expect(rows[0].error).not.toContain(host) + }) + + test("a call whose row could not be tracked reports false", () => { + spy.mockImplementation(() => { + throw new Error("sink down") + }) + expect(call("datamate", "error", "boom")).toBe(false) + }) + + test("the engine key in an unbound turn is not the workspace's; the turn's workspace, not the session's entry, names the row", () => { + expect( + trackWorkspaceToolCall({ + key: "datamate_x", + clientName: "datamate", + sessionID: "s1", + messageID: "m", + workspaceId: undefined, + startedAt: Date.now(), + outcome: "success", + }), + ).toBe(false) + expect(rows).toHaveLength(0) + // A replacement turn is current for the session, but this call ran for workspace 42. + Telemetry.setContext({ sessionId: "s1", projectId: "p1" }) + expect(call("datamate", "success")).toBe(true) + expect(rows[0].workspace_id).toBe(42) + Telemetry.setContext({ sessionId: "", projectId: "" }) + }) + + test("other MCP clients, standalone datamate servers and registry tool ids are not the workspace's engine", () => { + expect(call("github", "success")).toBe(false) + expect(call("filesystem", "error", new Error("x"))).toBe(false) + expect(call("datamate-acme", "success")).toBe(false) // a standalone datamate server the user configured + expect(call("datamates", "success")).toBe(false) + expect(rows).toHaveLength(0) + expect(Telemetry.workspaceUseOf("s1").integrations).toBe(0) + // The registry tool `datamate_manager` runs through Tool.define, never through this path, and its + // id is not the engine's client name either. + expect(isWorkspaceClient("datamate_manager")).toBe(false) + expect(isWorkspaceClient("datamate")).toBe(true) + expect(isWorkspaceClient("Datamate")).toBe(false) + }) +}) diff --git a/packages/opencode/test/memory/overlay-merge.test.ts b/packages/opencode/test/memory/overlay-merge.test.ts index 93ecae5eae..b77fd0970e 100644 --- a/packages/opencode/test/memory/overlay-merge.test.ts +++ b/packages/opencode/test/memory/overlay-merge.test.ts @@ -2,7 +2,7 @@ // Covers the injection-side merge of workspace memory into local memory: // which blocks survive, and the guard that keeps a cloud-sourced training // block from writing to the local store. -import { afterAll, afterEach, beforeEach, describe, expect, test } from "bun:test" +import { afterAll, afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" import path from "node:path" import os from "node:os" @@ -141,6 +141,41 @@ describe("the memory refresh tool", () => { expect(res.output).toContain("2 block(s)") }) + test("a block from the bound workspace's memory is injected under the workspace scope and counted", async () => { + const { Telemetry } = await import("../../src/altimate/telemetry") + // A project-scope block is the workspace's; a global cloud block is the account's and is + // injected too, but not counted as workspace memory. + listResponse = [ + remote("ws/one", "WORKSPACE BLOCK", { block_scope: "project", datamate_id: "7" }), + remote("acct/one", "GLOBAL BLOCK"), + ] + await hydrate(SES) + expect( + overlayBlocks(SES) + .map((b) => b.id) + .sort(), + ).toEqual(["acct/one", "ws/one"]) + const events: any[] = [] + const spy = spyOn(Telemetry, "track").mockImplementation((e) => { + events.push(e) + }) + try { + Telemetry.clearWorkspaceUse(SES) + const injected = await MemoryPrompt.inject(8000, { sessionID: SES }) + expect(injected).toContain("WORKSPACE BLOCK") + expect(injected).toContain("GLOBAL BLOCK") + const injection = events.find((e) => e.type === "memory_injection") + expect(injection).toBeDefined() + expect(injection.scopes_used).toContain("workspace") + expect(injection.scopes_used).toContain("global") + expect(injection.workspace_blocks).toBe(1) + expect(Telemetry.workspaceUseOf(SES).memory).toBe(1) + } finally { + spy.mockRestore() + Telemetry.clearWorkspaceUse(SES) + } + }) + test("says the workspace is unreachable rather than reporting an empty reload", async () => { listResponse = [remote("keep/one", "KEEP")] await hydrate(SES) @@ -280,7 +315,11 @@ describe("on-demand reload", () => { const { count, ok } = await refresh(SES) expect(ok).toBe(true) expect(count).toBe(2) - expect(overlayBlocks(SES).map((b) => b.id).sort()).toEqual(["warehouse/one", "warehouse/two"]) + expect( + overlayBlocks(SES) + .map((b) => b.id) + .sort(), + ).toEqual(["warehouse/one", "warehouse/two"]) }) test("a failed refresh keeps the memory the session already had", async () => { @@ -313,7 +352,7 @@ describe("on-demand reload", () => { await hydrate(SES) expect(overlayBlocks(SES).map((b) => b.id)).toEqual(["keep/me"]) - memoryEnabledCache.clear() // past the 60s positive-cache TTL + memoryEnabledCache.clear() // past the 60s positive-cache TTL const originalFetch = globalThis.fetch globalThis.fetch = (async (_i?: unknown, _n?: unknown) => { throw new Error("workspace unreachable") @@ -351,21 +390,22 @@ describe("on-demand reload", () => { firstCall = false await gate return new Response(JSON.stringify([remote("stale/one", "STALE")]), { - status: 200, headers: { "Content-Type": "application/json" }, + status: 200, + headers: { "Content-Type": "application/json" }, }) } return originalFetch(input, init) }) as typeof fetch try { - void hydrate(SES) // A — stalls on the gate + void hydrate(SES) // A — stalls on the gate await new Promise((r) => setTimeout(r, 20)) listResponse = [remote("fresh/one", "FRESH")] - const res = await refresh(SES) // B — completes first + const res = await refresh(SES) // B — completes first expect(res.status).toBe("loaded") expect(overlayBlocks(SES).map((b) => b.id)).toEqual(["fresh/one"]) - release?.() // A resolves late + release?.() // A resolves late await new Promise((r) => setTimeout(r, 50)) expect(overlayBlocks(SES).map((b) => b.id)).toEqual(["fresh/one"]) } finally { diff --git a/packages/opencode/test/telemetry/error-detail.test.ts b/packages/opencode/test/telemetry/error-detail.test.ts index 58daf34d5c..9eb92fb108 100644 --- a/packages/opencode/test/telemetry/error-detail.test.ts +++ b/packages/opencode/test/telemetry/error-detail.test.ts @@ -108,6 +108,6 @@ describe("error_fingerprint wiring", () => { test("prompt.ts emits through these helpers", async () => { const src = await Bun.file(new URL("../../src/session/prompt.ts", import.meta.url)).text() expect(src).toContain("...ErrorDetail.describeToolError(errorMsg)") - expect(src).toContain("Telemetry.track(ErrorDetail.buildErrorFingerprintEvent(sessionID, err))") + expect(src).toContain("Telemetry.track({ ...ErrorDetail.buildErrorFingerprintEvent(sessionID, err), ...turnWorkspace })") }) }) diff --git a/packages/opencode/test/tool/batch.test.ts b/packages/opencode/test/tool/batch.test.ts index f59fc9e3c3..5d9b7012bb 100644 --- a/packages/opencode/test/tool/batch.test.ts +++ b/packages/opencode/test/tool/batch.test.ts @@ -157,6 +157,36 @@ describe("BatchTool: file lessons", () => { }) }) +describe("BatchTool: inner tool context", () => { + test("carries the turn's workspace and token to the inner tools", async () => { + const tool = await getToolInfo() + let seen: { workspaceId?: number; turn?: number } | undefined + const registry = spyOn(ToolRegistry, "tools").mockResolvedValue([{ + id: "probe", + registrySource: undefined, + description: "captures its context", + parameters: Schema.Unknown, + execute: (_args: unknown, ctx: { workspaceId?: number; turn?: number }) => { + seen = { workspaceId: ctx.workspaceId, turn: ctx.turn } + return Effect.succeed({ title: "ok", output: "ok", metadata: {} }) + }, + }] as any) + const updatePart = Object.assign(async (part: Parameters[0]) => part, Session.updatePart) + const updates = spyOn(Session, "updatePart").mockImplementation(updatePart) + try { + const result = await tool.execute({ tool_calls: [{ tool: "probe", parameters: {} }] }, { + sessionID: SessionID.make("ses_batch_ctx"), messageID: MessageID.make("msg_batch_ctx"), agent: "build", + abort: AbortSignal.any([]), messages: [], extra: {}, workspaceId: 7, turn: 3, + } as any) + expect(result.metadata.successful).toBe(1) + expect(seen).toEqual({ workspaceId: 7, turn: 3 }) + } finally { + updates.mockRestore() + registry.mockRestore() + } + }) +}) + describe("BatchTool: schema validation", () => { test("rejects empty tool_calls array", async () => { const tool = await getToolInfo() diff --git a/packages/opencode/test/tool/skill-source.test.ts b/packages/opencode/test/tool/skill-source.test.ts index 3ede7dcf73..2355c98c53 100644 --- a/packages/opencode/test/tool/skill-source.test.ts +++ b/packages/opencode/test/tool/skill-source.test.ts @@ -18,6 +18,19 @@ describe("classifySkillSource", () => { expect(classifySkillSource("C:\\Users\\x\\.altimate\\builtin\\dbt-analyze\\SKILL.md")).toBe("builtin") }) + test("skills synced from the bound workspace (the managed _workspace folder) → workspace", () => { + expect(classifySkillSource("/work/repo/.altimate-code/skill/_workspace/dbt-review/SKILL.md")).toBe("workspace") + expect(classifySkillSource("C:\\work\\repo\\.altimate-code\\skill\\_workspace\\x\\SKILL.md")).toBe("workspace") + // a project skill that merely mentions the word is still the project's + expect(classifySkillSource("/work/repo/.altimate-code/skill/_workspace-notes/SKILL.md")).toBe("project") + // With the project root known, only this project's managed folder is the workspace's. + expect(classifySkillSource("/work/repo/.altimate-code/skill/_workspace/x/SKILL.md", "/work/repo")).toBe("workspace") + expect(classifySkillSource("/work/repo/.altimate-code/skill/_workspace/x/SKILL.md", "/work/repo/")).toBe( + "workspace", + ) + expect(classifySkillSource("/work/other/.altimate-code/skill/_workspace/x/SKILL.md", "/work/repo")).toBe("project") + }) + test("a third-party skill under some other node_modules is NOT tagged Altimate", () => { expect(classifySkillSource("/work/repo/node_modules/some-other-pkg/skills/x/SKILL.md")).toBe("project") }) diff --git a/packages/plugin/src/index.ts b/packages/plugin/src/index.ts index edfa0139df..42d3e1fdb8 100644 --- a/packages/plugin/src/index.ts +++ b/packages/plugin/src/index.ts @@ -271,14 +271,18 @@ export interface Hooks { input: { cwd: string; sessionID?: string; callID?: string }, output: { env: Record }, ) => Promise + // altimate_change start — `workspaceId`: the workspace of the turn the tool ran in, set by the + // trigger site, so a plugin's own rows name it without consulting the session's current turn + // (which may be a replacement by the time a cancelled turn's tool returns) "tool.execute.after"?: ( - input: { tool: string; sessionID: string; callID: string; args: any }, + input: { tool: string; sessionID: string; callID: string; args: any; workspaceId?: number }, output: { title: string output: string metadata: any }, ) => Promise + // altimate_change end "experimental.chat.messages.transform"?: ( input: {}, output: {